Skip to content

Commit 44382a5

Browse files
Dima-AndroidDima-Android
committed
Adding translation/translate submodule,
Updating submodules. TranslationLoader to check if extracted translator's JS code needs to be updated and updates it. bundle_translation.py zips translator's JS code and puts it into project's assets folder. Updating android.yml to support bundling of translation during CI build. Upping versionCode to 59
1 parent 9219fdc commit 44382a5

File tree

22 files changed

+1447
-24
lines changed

22 files changed

+1447
-24
lines changed

Diff for: ‎.github/workflows/android.yml

+6
Original file line numberDiff line numberDiff line change
@@ -45,6 +45,12 @@ jobs:
4545
- name: Execute bundle_translators.py
4646
run: python3 scripts/bundle_translators.py
4747

48+
- name: Grant execute permission for bundle_translation.py
49+
run: chmod +x scripts/bundle_translation.py
50+
51+
- name: Execute bundle_translation.py
52+
run: python3 scripts/bundle_translation.py
53+
4854
- name: Grant execute permission for gradlew
4955
run: chmod +x gradlew
5056

Diff for: ‎.gitmodules

+3
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,6 @@
11
[submodule "translators"]
22
path = translators
33
url = https://github.com/zotero/translators.git
4+
[submodule "translation/translate"]
5+
path = translation/translate
6+
url = https://github.com/zotero/translate.git

Diff for: ‎app/src/main/assets/timestamp.txt

+1-1
Original file line numberDiff line numberDiff line change
@@ -1 +1 @@
1-
1713182643
1+
1714480514

Diff for: ‎app/src/main/assets/translator.zip

-416 KB
Binary file not shown.

Diff for: ‎app/src/main/java/org/zotero/android/architecture/Defaults.kt

+9-9
Original file line numberDiff line numberDiff line change
@@ -40,9 +40,9 @@ open class Defaults @Inject constructor(
4040
private val activeLineWidth = "activeLineWidth"
4141
private val activeEraserSize = "activeEraserSize"
4242
private val shareExtensionIncludeTags = "shareExtensionIncludeTags"
43-
private val shouldUpdateTranslator = "shouldUpdateTranslator"
4443

4544
private val lastTimestamp = "lastTimestamp"
45+
private val lastTranslationCommitHash = "lastTranslationCommitHash"
4646
private val lastTranslatorCommitHash = "lastTranslatorCommitHash"
4747
private val lastTranslatorDeleted = "lastTranslatorDeleted"
4848
private val lastStylesCommitHash = "lastStylesCommitHash"
@@ -257,14 +257,6 @@ open class Defaults @Inject constructor(
257257
return sharedPreferences.getBoolean(wasPspdfkitInitialized, false)
258258
}
259259

260-
fun setShouldUpdateTranslator(newValue: Boolean) {
261-
sharedPreferences.edit { putBoolean(shouldUpdateTranslator, newValue) }
262-
}
263-
264-
fun shouldUpdateTranslator(): Boolean {
265-
return sharedPreferences.getBoolean(shouldUpdateTranslator, true)
266-
}
267-
268260
fun setLastTimestamp(newValue: Long) {
269261
sharedPreferences.edit { putLong(lastTimestamp, newValue) }
270262
}
@@ -281,6 +273,14 @@ open class Defaults @Inject constructor(
281273
return sharedPreferences.getString(lastTranslatorCommitHash, "") ?: ""
282274
}
283275

276+
fun setLastTranslationCommitHash(newValue: String) {
277+
sharedPreferences.edit { putString(lastTranslationCommitHash, newValue) }
278+
}
279+
280+
fun getLastTranslationCommitHash(): String {
281+
return sharedPreferences.getString(lastTranslationCommitHash, "") ?: ""
282+
}
283+
284284
fun getLastTranslatorDeleted(): Long {
285285
return sharedPreferences.getLong(lastTranslatorDeleted, 0L)
286286
}

Diff for: ‎app/src/main/java/org/zotero/android/sync/Controllers.kt

+3-1
Original file line numberDiff line numberDiff line change
@@ -20,6 +20,7 @@ import org.zotero.android.attachmentdownloader.AttachmentDownloader
2020
import org.zotero.android.database.DbWrapper
2121
import org.zotero.android.files.FileStore
2222
import org.zotero.android.screens.share.backgroundprocessor.BackgroundUploadProcessor
23+
import org.zotero.android.translator.loader.TranslationLoader
2324
import org.zotero.android.translator.loader.TranslatorsLoader
2425
import timber.log.Timber
2526
import javax.inject.Inject
@@ -44,6 +45,7 @@ class Controllers @Inject constructor(
4445
private val debugLogging: DebugLogging,
4546
private val crashReporter: CrashReporter,
4647
private val translatorsLoader: TranslatorsLoader,
48+
private val translationLoader: TranslationLoader,
4749
private val context: Context,
4850
) {
4951
private var sessionCancellable: Job? = null
@@ -75,7 +77,7 @@ class Controllers @Inject constructor(
7577
private fun updateTranslatorAndTranslatorItems() {
7678
coroutineScope.launch {
7779
try {
78-
translatorsLoader.updateTranslatorIfNeeded()
80+
translationLoader.updateTranslationIfNeeded()
7981
translatorsLoader.updateTranslatorItemsIfNeeded()
8082
} catch (e: Exception) {
8183
Timber.e(e, "Failed to update Translator or translation items")

Diff for: ‎app/src/main/java/org/zotero/android/translator/loader/TranslationLoader.kt

+171
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,171 @@
1+
package org.zotero.android.translator.loader
2+
3+
import android.content.Context
4+
import org.apache.commons.io.IOUtils
5+
import org.zotero.android.architecture.Defaults
6+
import org.zotero.android.files.FileStore
7+
import org.zotero.android.helpers.Unzipper
8+
import timber.log.Timber
9+
import javax.inject.Inject
10+
import javax.inject.Singleton
11+
12+
@Singleton
13+
class TranslationLoader @Inject constructor(
14+
private val context: Context,
15+
private val defaults: Defaults,
16+
private val unzipper: Unzipper,
17+
private val fileStore: FileStore,
18+
) {
19+
enum class UpdateType(val i: Int) {
20+
manual(1),
21+
initial(2),
22+
startup(3),
23+
notification(4),
24+
shareExtension(5);
25+
}
26+
27+
sealed class Error : Exception() {
28+
data class bundleLoading(val exception: Exception) : Error()
29+
object bundleMissing : Error()
30+
31+
val isBundleLoadingError: Boolean
32+
get() {
33+
return when (this) {
34+
is bundleLoading -> {
35+
true
36+
}
37+
38+
else -> {
39+
false
40+
}
41+
}
42+
}
43+
}
44+
45+
fun updateTranslationIfNeeded() {
46+
_update()
47+
}
48+
49+
private fun _update() {
50+
val type: UpdateType =
51+
if (defaults.getLastTimestamp() == 0L) {
52+
UpdateType.initial
53+
} else {
54+
UpdateType.startup
55+
}
56+
57+
Timber.i("TranslationLoader: update translation JS")
58+
try {
59+
checkFolderIntegrity(type = type)
60+
updateFromBundle()
61+
defaults.setLastTimestamp(System.currentTimeMillis() / 1000)
62+
} catch (error: Exception) {
63+
process(error = error, updateType = type)
64+
}
65+
}
66+
67+
private fun _updateTranslationFromBundle(forceUpdate: Boolean) {
68+
val hash = loadLastTranslationCommitHash()
69+
Timber.i("TranslationLoader: should update translation from bundle, forceUpdate=$forceUpdate; oldHash=${defaults.getLastTranslationCommitHash()}; newHash=$hash")
70+
if (!forceUpdate && defaults.getLastTranslationCommitHash() == hash) {
71+
return
72+
}
73+
Timber.i("TranslationLoader: update translation from bundle")
74+
updateTranslation()
75+
76+
defaults.setLastTranslationCommitHash(hash)
77+
}
78+
79+
private fun updateTranslation() {
80+
unzipper.unzipStream(
81+
zipInputStream = context.assets.open("translator.zip"),
82+
location = fileStore.translatorDirectory().absolutePath
83+
)
84+
}
85+
86+
private fun loadLastTranslationCommitHash(): String {
87+
return loadFromBundle(resource = "translation_commit_hash.txt", map = { it })
88+
}
89+
90+
91+
private inline fun <reified Result> loadFromBundle(
92+
resource: String,
93+
map: (String) -> Result
94+
): Result {
95+
try {
96+
val inputStream = context.assets.open(resource)
97+
val rawValue = IOUtils.toString(inputStream)
98+
return map(rawValue.trim().trim { it == '\n' })
99+
} catch (e: Exception) {
100+
Timber.e(e)
101+
throw Error.bundleMissing
102+
}
103+
}
104+
105+
private fun updateFromBundle() {
106+
try {
107+
_updateTranslationFromBundle(forceUpdate = false)
108+
val timestamp = loadLastTimestamp()
109+
if (timestamp > defaults.getLastTimestamp()) {
110+
defaults.setLastTimestamp(timestamp)
111+
return
112+
} else {
113+
return
114+
}
115+
116+
} catch (error: Exception) {
117+
Timber.e(error, "TranslatorsLoader: can't update from bundle")
118+
throw Error.bundleLoading(error)
119+
}
120+
}
121+
122+
private fun loadLastTimestamp(): Long {
123+
return loadFromBundle(resource = "timestamp.txt", map = {
124+
try {
125+
return it.toLong()
126+
} catch (e: Exception) {
127+
Timber.e(e)
128+
throw Error.bundleMissing
129+
}
130+
})
131+
}
132+
133+
private fun checkFolderIntegrity(type: UpdateType) {
134+
try {
135+
if (!fileStore.translatorDirectory().exists()) {
136+
if (type != UpdateType.initial) {
137+
Timber.e("TranslationLoader: translation directory was missing!")
138+
}
139+
fileStore.translatorDirectory().mkdirs()
140+
}
141+
142+
if (type == UpdateType.initial) {
143+
return
144+
}
145+
146+
val fileCount = fileStore.translatorDirectory().listFiles()?.size ?: 0
147+
148+
if (fileCount != 0) {
149+
return
150+
}
151+
152+
defaults.setLastTimestamp(0L)
153+
defaults.setLastTranslationCommitHash("")
154+
} catch (error: Exception) {
155+
Timber.e(error, "TranslationLoader: unable to restore folder integrity")
156+
throw error
157+
}
158+
159+
}
160+
161+
private fun process(error: Exception, updateType: UpdateType) {
162+
Timber.e(error, "TranslatorsLoader: error")
163+
164+
val isBundleLoadingError = (error as? Error)?.isBundleLoadingError == true
165+
if (!isBundleLoadingError) {
166+
return
167+
}
168+
//TODO show bundle load error dialog
169+
170+
}
171+
}

Diff for: ‎app/src/main/java/org/zotero/android/translator/loader/TranslatorsLoader.kt

-12
Original file line numberDiff line numberDiff line change
@@ -14,7 +14,6 @@ import org.zotero.android.architecture.coroutines.Dispatchers
1414
import org.zotero.android.database.DbWrapper
1515
import org.zotero.android.database.requests.SyncTranslatorsDbRequest
1616
import org.zotero.android.files.FileStore
17-
import org.zotero.android.helpers.Unzipper
1817
import org.zotero.android.screens.share.data.TranslatorMetadata
1918
import timber.log.Timber
2019
import java.io.File
@@ -30,7 +29,6 @@ class TranslatorsLoader @Inject constructor(
3029
private val context: Context,
3130
private val gson: Gson,
3231
private val defaults: Defaults,
33-
private val unzipper: Unzipper,
3432
private val itemsUnzipper: TranslatorItemsUnzipper,
3533
private val fileStore: FileStore,
3634
@BundleDataDb
@@ -245,16 +243,6 @@ class TranslatorsLoader @Inject constructor(
245243
return null
246244
}
247245

248-
fun updateTranslatorIfNeeded() {
249-
if (defaults.shouldUpdateTranslator()) {
250-
unzipper.unzipStream(
251-
zipInputStream = context.assets.open("translator.zip"),
252-
location = fileStore.translatorDirectory().absolutePath
253-
)
254-
defaults.setShouldUpdateTranslator(false)
255-
}
256-
}
257-
258246
private fun loadIndex(): List<TranslatorMetadata> {
259247
val inputStream = context.assets.open("translators/index.json")
260248
val type =

Diff for: ‎buildSrc/src/main/kotlin/BuildConfig.kt

+1-1
Original file line numberDiff line numberDiff line change
@@ -4,7 +4,7 @@ object BuildConfig {
44
const val compileSdkVersion = 34
55
const val targetSdk = 33
66

7-
val versionCode = 58 // Must be updated on every build
7+
val versionCode = 59 // Must be updated on every build
88
val version = Version(
99
major = 1,
1010
minor = 0,

Diff for: ‎scripts/bundle_translation.py

+34
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,34 @@
1+
import json
2+
import os
3+
import re
4+
import shutil
5+
import subprocess
6+
import time
7+
8+
def commit_hash_from_submodules(array):
9+
for line in array:
10+
if line.startswith("translation/translate"):
11+
return line.split()[1]
12+
13+
# Get bundle directory
14+
bundle_dir = os.path.join(os.path.abspath("."), "app" + os.sep + "src" + os.sep + "main" + os.sep + "assets")
15+
16+
if not os.path.isdir(bundle_dir):
17+
os.mkdir(bundle_dir)
18+
19+
# Get translation directory
20+
translators_dir = os.path.join(os.path.abspath("."), "translation")
21+
22+
if not os.path.isdir(translators_dir):
23+
raise Exception(translators_dir + " is not a directory. Call update_bundled_data.py first.")
24+
25+
# Store last commit hash from translation submodule
26+
submodules = subprocess.check_output(["git", "submodule", "foreach", "--recursive", "echo $path `git rev-parse HEAD`"]).decode("utf-8").splitlines()
27+
commit_hash = commit_hash_from_submodules(submodules)
28+
29+
with open(os.path.join(bundle_dir, "translation_commit_hash.txt"), "w") as f:
30+
f.write(str(commit_hash))
31+
32+
# Zip translator
33+
os.chdir(translators_dir)
34+
subprocess.check_call(['zip', '-r', os.path.join(bundle_dir, "translator.zip"), "."])

0 commit comments

Comments
 (0)