@ -1,16 +1,15 @@
package net.pokeranalytics.android.util.csv package net.pokeranalytics.android.util.csv
import android.content.Context
import android.net.Uri
import android.os.Handler import android.os.Handler
import android.os.Looper import android.os.Looper
import io.realm.Realm import io.realm.Realm
import net.pokeranalytics.android.model.realm.Session import net.pokeranalytics.android.exceptions.PAIllegalStateException
import net.pokeranalytics.android.util.extensions.count
import org.apache.commons.csv.CSVFormat import org.apache.commons.csv.CSVFormat
import org.apache.commons.csv.CSVParser
import org.apache.commons.csv.CSVRecord import org.apache.commons.csv.CSVRecord
import timber.log.Timber import timber.log.Timber
import java.io.FileReader import java.io.FileReader
import java.io.InputStream
import java.io.InputStreamReader import java.io.InputStreamReader
import java.io.Reader import java.io.Reader
@ -23,12 +22,12 @@ interface ImportDelegate {
/ * * / * *
* A CSVImporter is a class in charge of parsing a CSV file and processing it * A CSVImporter is a class in charge of parsing a CSV file and processing it
* When starting the parsing of a file , the instance will search for a CSVDescriptor , which describes * When starting the parsing of a file , the instance will search for a CSVDescriptor ,
* the format of a CSV file . * which describes the format of a CSV file .
* When finding a descriptor , the CSVImporter then continue to parse the file and delegates the parsing of each row * When finding a descriptor , the CSVImporter then continue to parse the file and
* to the CSVDescriptor * delegates the parsing of each row to the CSVDescriptor
* / * /
open class CSVImporter ( istream : InputStream ) { open class CSVImporter ( uri : Uri , var context : Context ) {
/ * * / * *
* The object being notified of the import progress * The object being notified of the import progress
@ -52,10 +51,11 @@ open class CSVImporter(istream: InputStream) {
* The path of the CSV file * The path of the CSV file
* / * /
private var path : String ? = null private var path : String ? = null
/ * * / * *
* The InputStream containing a file conten t * The Uri of the file to impor t
* / * /
private var inputStream : InputStream ? = istream private var uri : Uri ? = uri
/ * * / * *
* The current number of attempts at finding a valid CSVDescriptor * The current number of attempts at finding a valid CSVDescriptor
@ -85,97 +85,117 @@ open class CSVImporter(istream: InputStream) {
/ * * / * *
* The CSV parser * The CSV parser
* / * /
private lateinit var parser : CSVParser // private var parser: CSVParser? = null
val reader : Reader
get ( ) {
this . uri ?. let { uri ->
// it's required to open the stream each time we start a new parser
val inputStream = context . contentResolver . openInputStream ( uri )
return InputStreamReader ( inputStream )
}
return if ( this . path != null ) {
FileReader ( this . path )
} else {
throw PAIllegalStateException ( " No data source " )
}
}
/ * * / * *
* Constructs a CSVParser object and starts parsing the CSV * Constructs a CSVParser object and starts parsing the CSV
* / * /
fun start ( ) { fun start ( ) {
Timber . d ( " Starting import... " )
val realm = Realm . getDefaultInstance ( ) val realm = Realm . getDefaultInstance ( )
realm . beginTransaction ( )
var reader : Reader ? = null val descriptorsByDelimiter = ProductCSVDescriptors . all . groupBy { it . source . delimiter }
if ( this . path != null ) {
reader = FileReader ( this . path )
}
if ( this . inputStream != null ) {
reader = InputStreamReader ( this . inputStream )
}
this . parser = CSVFormat . DEFAULT . withAllowMissingColumnNames ( ) . parse ( reader ) for ( ( delimiter , descriptors ) in descriptorsByDelimiter ) {
Timber . d ( " Starting import... " ) if ( this . currentDescriptor != null ) {
break
}
realm . beginTransaction ( ) Timber . d ( " ====== Trying with delimiter ' $delimiter ' " )
val format = CSVFormat . DEFAULT
. withAllowMissingColumnNames ( )
. withDelimiter ( delimiter )
val parser = format . parse ( this . reader )
this . parser . forEachIndexed { index , record -> Timber . d ( " parse delim = ${format.delimiter} " )
parser . forEachIndexed { index , record ->
// Timber.d("line $index") // Timber.d("line $index")
this . notifyDelegate ( ) this . notifyDelegate ( )
if ( this . currentDescriptor == null ) { // find descriptor if ( this . currentDescriptor == null ) { // find descriptor
this . currentDescriptor = this . findDescriptor ( record ) this . currentDescriptor = this . findDescriptor ( record , descriptors )
this . currentDescriptor ?. hasMatched ( realm , record ) this . currentDescriptor ?. hasMatched ( realm , record )
if ( this . currentDescriptor == null ) { if ( this . currentDescriptor == null ) {
if ( record . size ( ) >= VALID _RECORD _COLUMNS ) { if ( record . size ( ) >= VALID _RECORD _COLUMNS ) {
this . descriptorFindingAttempts ++ this . descriptorFindingAttempts ++
} }
if ( this . descriptorFindingAttempts >= VALID _RECORD _ATTEMPTS _BEFORE _THROWING _EXCEPTION ) { if ( this . descriptorFindingAttempts >= VALID _RECORD _ATTEMPTS _BEFORE _THROWING _EXCEPTION ) {
realm . cancelTransaction ( ) realm . cancelTransaction ( )
realm . close ( ) realm . close ( )
throw ImportException ( " This type of file is not supported " ) throw ImportException ( " This type of file is not supported " )
}
} }
}
} else { // parse } else { // parse
// batch commit // batch commit
val parsingIndex = index + 1 val parsingIndex = index + 1
if ( parsingIndex % COMMIT _FREQUENCY == 0 ) { if ( parsingIndex % COMMIT _FREQUENCY == 0 ) {
Timber . d ( " ****** committing at $parsingIndex sessions... " ) Timber . d ( " ****** committing at $parsingIndex sessions... " )
realm . commitTransaction ( ) realm . commitTransaction ( )
realm . beginTransaction ( ) realm . beginTransaction ( )
} }
this . currentDescriptor ?. let { this . currentDescriptor ?. let {
if ( record . size ( ) == 0 ) { if ( record . size ( ) == 0 ) {
this . usedDescriptors . add ( it ) this . usedDescriptors . add ( it )
this . currentDescriptor = this . currentDescriptor =
null // reset descriptor when encountering an empty line (multiple descriptors can be found in a single file) null // reset descriptor when encountering an empty line (multiple descriptors can be found in a single file)
this . descriptorFindingAttempts = 0 this . descriptorFindingAttempts = 0
} else { } else {
try { try {
val count = it . parse ( realm , record ) val count = it . parse ( realm , record , this . context )
this . importedRecords += count this . importedRecords += count
this . totalParsedRecords ++ this . totalParsedRecords ++
this . notifyDelegate ( ) this . notifyDelegate ( )
} catch ( e : Exception ) { } catch ( e : Exception ) {
this . delegate ?. exceptionCaught ( e ) this . delegate ?. exceptionCaught ( e )
} }
}
} ?: run {
realm . cancelTransaction ( )
realm . close ( )
throw ImportException ( " CSVDescriptor should never be null here " )
} }
} ?: run {
realm . cancelTransaction ( )
realm . close ( )
throw ImportException ( " CSVDescriptor should never be null here " )
} }
} }
parser . close ( )
} }
Timber . d ( " Ending import... " )
this . notifyDelegate ( ) this . notifyDelegate ( )
realm . commitTransaction ( ) realm . commitTransaction ( )
Timber . d ( " Ending import... " )
realm . close ( ) realm . close ( )
} }
@ -188,9 +208,9 @@ open class CSVImporter(istream: InputStream) {
/ * * / * *
* Search for a descriptor in the list of managed formats * Search for a descriptor in the list of managed formats
* / * /
private fun findDescriptor ( record : CSVRecord ) : CSVDescriptor ? { private fun findDescriptor ( record : CSVRecord , descriptors : List < CSVDescriptor > ) : CSVDescriptor ? {
Pro ductCSVD escriptors. all . forEach { descriptor -> descriptors . forEach { descriptor ->
if ( descriptor . matches ( record ) ) { if ( descriptor . matches ( record ) ) {
this . currentDescriptor = descriptor this . currentDescriptor = descriptor
Timber . d ( " Identified source: ${descriptor.source} " ) Timber . d ( " Identified source: ${descriptor.source} " )
@ -201,7 +221,7 @@ open class CSVImporter(istream: InputStream) {
} }
fun save ( realm : Realm ) { fun save ( realm : Realm ) {
this . parser . close ( ) // this.parser?.close( )
realm . refresh ( ) realm . refresh ( )
this . currentDescriptor ?. save ( realm ) this . currentDescriptor ?. save ( realm )
@ -211,7 +231,7 @@ open class CSVImporter(istream: InputStream) {
} }
fun cancel ( realm : Realm ) { fun cancel ( realm : Realm ) {
this . parser . close ( ) // this.parser?.close( )
realm . refresh ( ) realm . refresh ( )
this . currentDescriptor ?. cancel ( realm ) this . currentDescriptor ?. cancel ( realm )
