Create dictionaries from user-provided wordlists

app/src/main/java/helium314/keyboard/latin/dictionary/DictionaryFactory.kt

@@ -74,11 +74,7 @@ object DictionaryFactory {
     }
 
     @JvmStatic
-    fun getDictionary(
-        file: File,
-        locale: Locale
-    ): Dictionary? {
-        if (!file.isFile) return null
+    fun getDictionary(file: File, locale: Locale): Dictionary? {
         val header = DictionaryInfoUtils.getDictionaryFileHeaderOrNull(file)
         if (header == null) {
             killDictionary(file)

app/src/main/java/helium314/keyboard/latin/dictionary/ExpandableBinaryDictionary.java

@@ -275,13 +275,15 @@ public void addUnigramEntry(final String word, final int frequency,
                 shortcutFreq, isNotAWord, isPossiblyOffensive, timestamp));
     }
 
-    protected void addUnigramLocked(final String word, final int frequency,
+    protected boolean addUnigramLocked(final String word, final int frequency,
             final String shortcutTarget, final int shortcutFreq, final boolean isNotAWord,
             final boolean isPossiblyOffensive, final int timestamp) {
         if (!mBinaryDictionary.addUnigramEntry(word, frequency, shortcutTarget, shortcutFreq,
                 false /* isBeginningOfSentence */, isNotAWord, isPossiblyOffensive, timestamp)) {
             Log.e(TAG, "Cannot add unigram entry. word: " + word);
+            return false;
         }
+        return true;
     }
 
     /**

app/src/main/java/helium314/keyboard/latin/dictionary/UserAddedDictionary.kt

@@ -0,0 +1,84 @@
+package helium314.keyboard.latin.dictionary
+
+import android.content.Context
+import com.android.inputmethod.latin.BinaryDictionary
+import helium314.keyboard.latin.makedict.DictionaryHeader
+import helium314.keyboard.latin.utils.DictionaryInfoUtils
+import helium314.keyboard.latin.utils.Log
+import java.io.File
+import java.util.Locale
+
+// todo
+//  how to add to correct directory?
+//   is it enough to provide the file?
+//   use user suffix?
+//  parse "everything"
+//   bigram
+//   shortcut
+//  ui for adding, with feedback
+//  ui for managing
+//   should show up in dictionaries screen
+//   should have a full header
+//  backup/restore
+class UserAddedDictionary(context: Context, locale: Locale, name: String) : ExpandableBinaryDictionary(
+    context,
+    name,
+    locale,
+    name,
+    File(DictionaryInfoUtils.getCacheDirectoryForLocale(locale, context)!!, name + "_" + DictionaryInfoUtils.USER_DICTIONARY_SUFFIX)
+) {
+    var content: List<String>? = null
+        private set
+    var added = 0
+    var failed = 0
+
+    fun setContents(contents: List<String>) {
+        Log.i(TAG, "setting new contents for ")
+        content = contents
+        setNeedsToRecreate()
+        reloadDictionaryIfRequired()
+    }
+
+    // todo: after around 350k words this becomes slow, and fails to add more words a bit later
+    //  split at 300k words? not nice for query performance for large dicts i guess
+    //  just tell the user that it's not working?
+    //  ideally we'd create an actual .dict file, here we also have some unknown but larger limit
+    override fun loadInitialContentsLocked() {
+        added = 0
+        failed = 0
+        content?.forEach { line ->
+            if (!line.trim().startsWith("word")) return@forEach
+            val split = line.split(",").map { it.trim() }
+            val success = addUnigramLocked(
+                split.first { it.startsWith("word=") }.substringAfter("word="),
+                split.first { it.startsWith("f=") }.substringAfter("f=").toInt(),
+                null,
+                0,
+                split.contains("not_a_word=true"),
+                split.contains("possibly_offensive=true"),
+                BinaryDictionary.NOT_A_VALID_TIMESTAMP
+            )
+            if (success) added++ else failed++
+            runGCIfRequiredLocked(true)
+        }
+        content = null
+        Log.i(TAG, "added $added entries, could not add $failed entries")
+    }
+
+    companion object {
+        private val TAG = UserAddedDictionary::class.java.simpleName
+
+        fun tryParseHeader(file: File): DictionaryHeader? {
+            val lines = file.readLines()
+            if (lines.size < 2) return null
+            if (!lines[1].trim().startsWith("word=", true) || !lines[1].contains("f=", true)) {
+                Log.e(TAG, "tryParseHeader: second line not a dictionary line: ${lines[1]}")
+                return null // not the right format (should be extended though, why not just accept word lists, or word + frequency)
+            }
+
+            return if (lines[0].trim().startsWith("dictionary", true))
+                runCatching { DictionaryHeader.fromString(lines[0]) }.getOrNull()
+            else DictionaryHeader.createEmptyHeader()
+        }
+    }
+}

app/src/main/java/helium314/keyboard/latin/makedict/DictionaryHeader.kt

@@ -11,6 +11,7 @@ import helium314.keyboard.latin.makedict.FormatSpec.DictionaryOptions
 import java.text.DateFormat
 import java.util.Date
 import java.util.Locale
+import kotlin.jvm.Throws
 
 /**
  * Class representing dictionary header.
@@ -58,5 +59,17 @@ class DictionaryHeader(
         const val MAX_TRIGRAM_COUNT_KEY = "MAX_TRIGRAM_ENTRY_COUNT"
         const val ATTRIBUTE_VALUE_TRUE = "1"
         const val CODE_POINT_TABLE_KEY = "codePointTable"
+
+        @Throws(UnsupportedFormatException::class)
+        fun fromString(string: String): DictionaryHeader {
+            val split = string.split(",").map { it.trim() }.filter { "=" in it }
+            val map = split.associateTo(HashMap()) { it.split("=").let { it[0] to it[1] } }
+            return DictionaryHeader(DictionaryOptions(map))
+        }
+
+        fun createEmptyHeader(): DictionaryHeader {
+            val map = hashMapOf(DICTIONARY_LOCALE_KEY to "", DICTIONARY_VERSION_KEY to "", DICTIONARY_ID_KEY to "")
+            return DictionaryHeader(DictionaryOptions(map))
+        }
     }
 }

app/src/main/java/helium314/keyboard/settings/FilePicker.kt

@@ -2,6 +2,7 @@
 package helium314.keyboard.settings
 
 import android.app.Activity
+import android.content.Context
 import android.content.Intent
 import android.net.Uri
 import android.provider.OpenableColumns
@@ -13,11 +14,16 @@ import androidx.compose.runtime.Composable
 import androidx.compose.runtime.getValue
 import androidx.compose.runtime.mutableStateOf
 import androidx.compose.runtime.remember
+import androidx.compose.runtime.saveable.rememberSaveable
 import androidx.compose.runtime.setValue
 import androidx.compose.ui.platform.LocalContext
 import androidx.compose.ui.res.stringResource
 import helium314.keyboard.latin.R
 import helium314.keyboard.latin.common.FileUtils
+import helium314.keyboard.latin.dictionary.UserAddedDictionary
+import helium314.keyboard.latin.makedict.DictionaryHeader
+import helium314.keyboard.latin.makedict.FormatSpec
+import helium314.keyboard.latin.utils.DictionaryInfoUtils
 import helium314.keyboard.latin.utils.LayoutUtilsCustom
 import helium314.keyboard.latin.utils.getActivity
 import helium314.keyboard.settings.dialogs.InfoDialog
@@ -46,17 +52,11 @@ fun layoutFilePicker(
     var errorDialog by remember { mutableStateOf(false) }
     val loadFilePicker = filePicker { uri ->
         val cr = ctx.getActivity()?.contentResolver ?: return@filePicker
-        val name = cr.query(uri, null, null, null, null)?.use { c ->
-            if (!c.moveToFirst()) return@use null
-            val index = c.getColumnIndex(OpenableColumns.DISPLAY_NAME)
-            if (index < 0) null
-            else c.getString(index)
-        }
         cr.openInputStream(uri)?.use {
             val content = it.reader().readText()
             errorDialog = !LayoutUtilsCustom.checkLayout(content, ctx)
             if (!errorDialog)
-                onSuccess(content, name)
+                onSuccess(content, getNameFromUri(ctx, uri))
         }
     }
     if (errorDialog)
@@ -68,18 +68,46 @@ fun layoutFilePicker(
 fun dictionaryFilePicker(mainLocale: Locale?): ManagedActivityResultLauncher<Intent, ActivityResult> {
     val ctx = LocalContext.current
     val cachedDictionaryFile = File(ctx.cacheDir?.path + File.separator + "temp_dict")
-    var done by remember { mutableStateOf(false) }
+    var name by rememberSaveable { mutableStateOf<String?>(null) }
     val picker = filePicker { uri ->
         cachedDictionaryFile.delete()
         FileUtils.copyContentUriToNewFile(uri, ctx, cachedDictionaryFile)
-        done = true
+        name = getNameFromUri(ctx, uri) ?: "dict"
+    }
+    if (name != null) {
+        val header = DictionaryInfoUtils.getDictionaryFileHeaderOrNull(cachedDictionaryFile, 0, cachedDictionaryFile.length())
+        val textHeader by lazy { UserAddedDictionary.tryParseHeader(cachedDictionaryFile) } // todo: got a "No such file or directory" exception here
+        if (header != null) {
+            NewDictionaryDialog(
+                onDismissRequest = { name = null },
+                cachedDictionaryFile,
+                mainLocale,
+                header,
+                false
+            )
+        } else if (textHeader != null) {
+            if (textHeader!!.mIdString == "") // no header in file
+                textHeader!!.mDictionaryOptions.mAttributes[DictionaryHeader.DICTIONARY_ID_KEY] = name
+            NewDictionaryDialog(
+                onDismissRequest = { name = null },
+                cachedDictionaryFile,
+                mainLocale,
+                DictionaryHeader(FormatSpec.DictionaryOptions(textHeader!!.mDictionaryOptions.mAttributes)),
+                true
+            )
+        } else {
+            InfoDialog(stringResource(R.string.dictionary_file_error)) { name = null }
+        }
     }
-    if (done)
-        NewDictionaryDialog(
-            onDismissRequest = { done = false },
-            cachedDictionaryFile,
-            mainLocale
-        )
 
     return picker
 }
+
+private fun getNameFromUri(context: Context, uri: Uri): String? {
+    return context.getActivity()?.contentResolver?.query(uri, null, null, null, null)?.use { c ->
+        if (!c.moveToFirst()) return@use null
+        val index = c.getColumnIndex(OpenableColumns.DISPLAY_NAME)
+        if (index < 0) null
+        else c.getString(index)
+    }
+}

app/src/main/java/helium314/keyboard/settings/SettingsActivity.kt

@@ -39,11 +39,13 @@ import helium314.keyboard.latin.common.FileUtils
 import helium314.keyboard.latin.define.DebugFlags
 import helium314.keyboard.latin.settings.Settings
 import helium314.keyboard.latin.utils.DeviceProtectedUtils
+import helium314.keyboard.latin.utils.DictionaryInfoUtils
 import helium314.keyboard.latin.utils.ExecutorUtils
 import helium314.keyboard.latin.utils.UncachedInputMethodManagerUtils
 import helium314.keyboard.latin.utils.cleanUnusedMainDicts
 import helium314.keyboard.latin.utils.prefs
 import helium314.keyboard.settings.dialogs.ConfirmationDialog
+import helium314.keyboard.settings.dialogs.InfoDialog
 import helium314.keyboard.settings.dialogs.NewDictionaryDialog
 import kotlinx.coroutines.flow.MutableStateFlow
 import java.io.BufferedOutputStream
@@ -138,11 +140,17 @@ open class SettingsActivity : ComponentActivity(), SharedPreferences.OnSharedPre
                         }
                     }
                     if (dictUri != null) {
-                        NewDictionaryDialog(
-                            onDismissRequest = { dictUriFlow.value = null },
-                            cachedFile = cachedDictionaryFile,
-                            mainLocale = null
-                        )
+                        val header = DictionaryInfoUtils.getDictionaryFileHeaderOrNull(cachedDictionaryFile, 0, cachedDictionaryFile.length())
+                        if (header == null)
+                            InfoDialog(stringResource(R.string.dictionary_file_error)) { dictUriFlow.value = null }
+                        else
+                            NewDictionaryDialog(
+                                onDismissRequest = { dictUriFlow.value = null },
+                                cachedFile = cachedDictionaryFile,
+                                mainLocale = null,
+                                header = header,
+                                isTextFile = false
+                            )
                     }
                 }
             }

app/src/main/java/helium314/keyboard/settings/dialogs/DictionaryDialog.kt

@@ -100,7 +100,7 @@ fun DictionaryDialog(
         onNeutral = {
             val intent = Intent(Intent.ACTION_OPEN_DOCUMENT)
                 .addCategory(Intent.CATEGORY_OPENABLE)
-                .setType("application/octet-stream")
+                .setType("*/*")
             picker.launch(intent)
         }
     )
@@ -135,7 +135,7 @@ private fun DictionaryDetails(dict: File) {
         ConfirmationDialog(
             onDismissRequest = { showDeleteDialog = false },
             confirmButtonText = stringResource(R.string.remove),
-            onConfirmed = { dict.delete() },
+            onConfirmed = { dict.deleteRecursively() },
             content = { Text(stringResource(R.string.remove_dictionary_message, type))}
         )
 }