@ -1,16 +1,15 @@
package net.pokeranalytics.android.util.csv
import android.content.Context
import android.net.Uri
import android.os.Handler
import android.os.Looper
import io.realm.Realm
import net.pokeranalytics.android.model.realm.Session
import net.pokeranalytics.android.util.extensions.count
import net.pokeranalytics.android.exceptions.PAIllegalStateException
import org.apache.commons.csv.CSVFormat
import org.apache.commons.csv.CSVParser
import org.apache.commons.csv.CSVRecord
import timber.log.Timber
import java.io.FileReader
import java.io.InputStream
import java.io.InputStreamReader
import java.io.Reader
@ -23,12 +22,12 @@ interface ImportDelegate {
/ * *
* A CSVImporter is a class in charge of parsing a CSV file and processing it
* When starting the parsing of a file , the instance will search for a CSVDescriptor , which describes
* the format of a CSV file .
* When finding a descriptor , the CSVImporter then continue to parse the file and delegates the parsing of each row
* to the CSVDescriptor
* When starting the parsing of a file , the instance will search for a CSVDescriptor ,
* which describes the format of a CSV file .
* When finding a descriptor , the CSVImporter then continue to parse the file and
* delegates the parsing of each row to the CSVDescriptor
* /
open class CSVImporter ( istream : InputStream ) {
open class CSVImporter ( uri : Uri , var context : Context ) {
/ * *
* The object being notified of the import progress
@ -52,10 +51,11 @@ open class CSVImporter(istream: InputStream) {
* The path of the CSV file
* /
private var path : String ? = null
/ * *
* The InputStream containing a file conten t
* The Uri of the file to impor t
* /
private var inputStream : InputStream ? = istream
private var uri : Uri ? = uri
/ * *
* The current number of attempts at finding a valid CSVDescriptor
@ -85,97 +85,117 @@ open class CSVImporter(istream: InputStream) {
/ * *
* The CSV parser
* /
private lateinit var parser : CSVParser
// private var parser: CSVParser? = null
val reader : Reader
get ( ) {
this . uri ?. let { uri ->
// it's required to open the stream each time we start a new parser
val inputStream = context . contentResolver . openInputStream ( uri )
return InputStreamReader ( inputStream )
}
return if ( this . path != null ) {
FileReader ( this . path )
} else {
throw PAIllegalStateException ( " No data source " )
}
}
/ * *
* Constructs a CSVParser object and starts parsing the CSV
* /
fun start ( ) {
Timber . d ( " Starting import... " )
val realm = Realm . getDefaultInstance ( )
realm . beginTransaction ( )
var reader : Reader ? = null
if ( this . path != null ) {
reader = FileReader ( this . path )
}
if ( this . inputStream != null ) {
reader = InputStreamReader ( this . inputStream )
}
val descriptorsByDelimiter = ProductCSVDescriptors . all . groupBy { it . source . delimiter }
this . parser = CSVFormat . DEFAULT . withAllowMissingColumnNames ( ) . parse ( reader )
for ( ( delimiter , descriptors ) in descriptorsByDelimiter ) {
Timber . d ( " Starting import... " )
if ( this . currentDescriptor != null ) {
break
}
realm . beginTransaction ( )
Timber . d ( " ====== Trying with delimiter ' $delimiter ' " )
val format = CSVFormat . DEFAULT
. withAllowMissingColumnNames ( )
. withDelimiter ( delimiter )
val parser = format . parse ( this . reader )
this . parser . forEachIndexed { index , record ->
Timber . d ( " parse delim = ${format.delimiter} " )
parser . forEachIndexed { index , record ->
// Timber.d("line $index")
this . notifyDelegate ( )
this . notifyDelegate ( )
if ( this . currentDescriptor == null ) { // find descriptor
this . currentDescriptor = this . findDescriptor ( record )
if ( this . currentDescriptor == null ) { // find descriptor
this . currentDescriptor = this . findDescriptor ( record , descriptors )
this . currentDescriptor ?. hasMatched ( realm , record )
this . currentDescriptor ?. hasMatched ( realm , record )
if ( this . currentDescriptor == null ) {
if ( this . currentDescriptor == null ) {
if ( record . size ( ) >= VALID _RECORD _COLUMNS ) {
this . descriptorFindingAttempts ++
}
if ( this . descriptorFindingAttempts >= VALID _RECORD _ATTEMPTS _BEFORE _THROWING _EXCEPTION ) {
realm . cancelTransaction ( )
realm . close ( )
throw ImportException ( " This type of file is not supported " )
if ( record . size ( ) >= VALID _RECORD _COLUMNS ) {
this . descriptorFindingAttempts ++
}
if ( this . descriptorFindingAttempts >= VALID _RECORD _ATTEMPTS _BEFORE _THROWING _EXCEPTION ) {
realm . cancelTransaction ( )
realm . close ( )
throw ImportException ( " This type of file is not supported " )
}
}
}
} else { // parse
} else { // parse
// batch commit
val parsingIndex = index + 1
if ( parsingIndex % COMMIT _FREQUENCY == 0 ) {
Timber . d ( " ****** committing at $parsingIndex sessions... " )
realm . commitTransaction ( )
realm . beginTransaction ( )
}
// batch commit
val parsingIndex = index + 1
if ( parsingIndex % COMMIT _FREQUENCY == 0 ) {
Timber . d ( " ****** committing at $parsingIndex sessions... " )
realm . commitTransaction ( )
realm . beginTransaction ( )
}
this . currentDescriptor ?. let {
if ( record . size ( ) == 0 ) {
this . usedDescriptors . add ( it )
this . currentDescriptor =
null // reset descriptor when encountering an empty line (multiple descriptors can be found in a single file)
this . descriptorFindingAttempts = 0
} else {
this . currentDescriptor ?. let {
if ( record . size ( ) == 0 ) {
this . usedDescriptors . add ( it )
this . currentDescriptor =
null // reset descriptor when encountering an empty line (multiple descriptors can be found in a single file)
this . descriptorFindingAttempts = 0
} else {
try {
val count = it . parse ( realm , record )
try {
val count = it . parse ( realm , record , this . context )
this . importedRecords += count
this . totalParsedRecords ++
this . importedRecords += count
this . totalParsedRecords ++
this . notifyDelegate ( )
this . notifyDelegate ( )
} catch ( e : Exception ) {
this . delegate ?. exceptionCaught ( e )
}
} catch ( e : Exception ) {
this . delegate ?. exceptionCaught ( e )
}
}
} ?: run {
realm . cancelTransaction ( )
realm . close ( )
throw ImportException ( " CSVDescriptor should never be null here " )
}
} ?: run {
realm . cancelTransaction ( )
realm . close ( )
throw ImportException ( " CSVDescriptor should never be null here " )
}
}
parser . close ( )
}
Timber . d ( " Ending import... " )
this . notifyDelegate ( )
realm . commitTransaction ( )
Timber . d ( " Ending import... " )
realm . close ( )
}
@ -188,9 +208,9 @@ open class CSVImporter(istream: InputStream) {
/ * *
* Search for a descriptor in the list of managed formats
* /
private fun findDescriptor ( record : CSVRecord ) : CSVDescriptor ? {
private fun findDescriptor ( record : CSVRecord , descriptors : List < CSVDescriptor > ) : CSVDescriptor ? {
Pro ductCSVD escriptors. all . forEach { descriptor ->
descriptors . forEach { descriptor ->
if ( descriptor . matches ( record ) ) {
this . currentDescriptor = descriptor
Timber . d ( " Identified source: ${descriptor.source} " )
@ -201,7 +221,7 @@ open class CSVImporter(istream: InputStream) {
}
fun save ( realm : Realm ) {
this . parser . close ( )
// this.parser?.close( )
realm . refresh ( )
this . currentDescriptor ?. save ( realm )
@ -211,7 +231,7 @@ open class CSVImporter(istream: InputStream) {
}
fun cancel ( realm : Realm ) {
this . parser . close ( )
// this.parser?.close( )
realm . refresh ( )
this . currentDescriptor ?. cancel ( realm )
