diff --git a/build.gradle b/build.gradle index 71f536ec2..865f4af67 100644 --- a/build.gradle +++ b/build.gradle @@ -465,6 +465,13 @@ def getRunTestsBuildArguments = { taskName -> if (onlyX86) { arguments.add("-PonlyX86") } + // App file modes, see test-app/tools/appfiles-test-mode.init.gradle + if (project.hasProperty("loadCodeFromApk")) { + arguments.add("-PloadCodeFromApk") + } + if (project.hasProperty("storeCodeUncompressed")) { + arguments.add("-PstoreCodeUncompressed") + } if (project.hasProperty("abis")) { arguments.add("-Pabis=${project.property('abis')}") } diff --git a/test-app/app/build.gradle b/test-app/app/build.gradle index d90f7ffe0..acc94c8c3 100644 --- a/test-app/app/build.gradle +++ b/test-app/app/build.gradle @@ -216,9 +216,31 @@ def computeNamespace = { -> return "com.tns.testapplication" } +/* + * With "android": { "storeCodeUncompressed": true } in app/package.json, code + * is stored in the APK uncompressed, so loadCodeFromApk reads it without + * inflating. noCompress matches by file-name suffix across the whole APK, not + * only under assets/app, so the metadata streams are named in full. + */ +project.ext.nsUncompressedCodeSuffixes = [".js", ".mjs", ".cjs", ".map", + "treeNodeStream.dat", "treeStringsStream.dat", "treeValueStream.dat"] + +def computeStoreCodeUncompressed = { -> + def appPackageJsonFile = file("${APP_DIR}/$PACKAGE_JSON") + if (!appPackageJsonFile.exists()) { + return false + } + def packageJsonMap = new JsonSlurper().parseText(appPackageJsonFile.getText("UTF-8")) + return packageJsonMap?.android?.storeCodeUncompressed == true +} + android { namespace = computeNamespace() + if (computeStoreCodeUncompressed()) { + androidResources.noCompress.addAll(project.ext.nsUncompressedCodeSuffixes) + } + applyBeforePluginGradleConfiguration() kotlinOptions { diff --git a/test-app/app/src/main/AndroidManifest.xml b/test-app/app/src/main/AndroidManifest.xml index f6896f5df..860e5be1c 100644 --- a/test-app/app/src/main/AndroidManifest.xml +++ b/test-app/app/src/main/AndroidManifest.xml @@ -23,6 +23,9 @@ + diff --git a/test-app/app/src/main/assets/app/mainpage.js b/test-app/app/src/main/assets/app/mainpage.js index 3996a5d75..e1dd76a75 100644 --- a/test-app/app/src/main/assets/app/mainpage.js +++ b/test-app/app/src/main/assets/app/mainpage.js @@ -115,6 +115,9 @@ require('./tests/testCreateRequire'); require('./tests/testNodeUrlModule'); require('./tests/testImportMetaResolution'); require('./tests/testWorkerEsmEntry'); +require('./tests/testCodeCache'); +require('./tests/testAppFileLayers'); +require('./tests/testLoadCodeFromApk'); // Worker wrapper reachability across GC (strong while running, collectable after) require('./tests/testWorkerLifetime'); // Java->JS calls route to the runtime that created the object diff --git a/test-app/app/src/main/assets/app/package.json b/test-app/app/src/main/assets/app/package.json index 2ca148662..978de3c20 100644 --- a/test-app/app/src/main/assets/app/package.json +++ b/test-app/app/src/main/assets/app/package.json @@ -10,7 +10,9 @@ "forceLog": false, "suppressCallJSMethodExceptions": false, "enableLineBreakpoints": false, - "enableMultithreadedJavascript": true + "enableMultithreadedJavascript": true, + "codeCache": true, + "alwaysExtract": ["tests/alwaysExtracted/**"] }, "discardUncaughtJsExceptions": false, "security": { diff --git a/test-app/app/src/main/assets/app/tests/alwaysExtracted/kept.js b/test-app/app/src/main/assets/app/tests/alwaysExtracted/kept.js new file mode 100644 index 000000000..3b3205cf9 --- /dev/null +++ b/test-app/app/src/main/assets/app/tests/alwaysExtracted/kept.js @@ -0,0 +1 @@ +module.exports.value = 'kept'; diff --git a/test-app/app/src/main/assets/app/tests/appFileLayers/target.js b/test-app/app/src/main/assets/app/tests/appFileLayers/target.js new file mode 100644 index 000000000..d8bf4a860 --- /dev/null +++ b/test-app/app/src/main/assets/app/tests/appFileLayers/target.js @@ -0,0 +1,2 @@ +module.exports.value = 'target'; +module.exports.filename = __filename; diff --git a/test-app/app/src/main/assets/app/tests/codeCacheLoadWorker.js b/test-app/app/src/main/assets/app/tests/codeCacheLoadWorker.js new file mode 100644 index 000000000..07be2cfe5 --- /dev/null +++ b/test-app/app/src/main/assets/app/tests/codeCacheLoadWorker.js @@ -0,0 +1,16 @@ +onmessage = function (msg) { + var data = msg.data; + var loaded; + try { + loaded = data.isModule + ? import(data.path).then(function (ns) { return ns.value; }) + : Promise.resolve(require(data.path).value); + } catch (e) { + loaded = Promise.reject(e); + } + loaded.then(function (value) { + postMessage({ value: value }); + }, function (e) { + postMessage({ error: String(e) }); + }); +}; diff --git a/test-app/app/src/main/assets/app/tests/loadCodeFromApk/aliasTarget.mjs b/test-app/app/src/main/assets/app/tests/loadCodeFromApk/aliasTarget.mjs new file mode 100644 index 000000000..d70bd7e8c --- /dev/null +++ b/test-app/app/src/main/assets/app/tests/loadCodeFromApk/aliasTarget.mjs @@ -0,0 +1 @@ +export const value = 'alias'; diff --git a/test-app/app/src/main/assets/app/tests/loadCodeFromApk/liveSyncOverride.js b/test-app/app/src/main/assets/app/tests/loadCodeFromApk/liveSyncOverride.js new file mode 100644 index 000000000..8b6278684 --- /dev/null +++ b/test-app/app/src/main/assets/app/tests/loadCodeFromApk/liveSyncOverride.js @@ -0,0 +1 @@ +module.exports.value = 'apk'; diff --git a/test-app/app/src/main/assets/app/tests/testAppFileLayers.js b/test-app/app/src/main/assets/app/tests/testAppFileLayers.js new file mode 100644 index 000000000..8727503e5 --- /dev/null +++ b/test-app/app/src/main/assets/app/tests/testAppFileLayers.js @@ -0,0 +1,42 @@ +// Layers come from com.tns.tests.TestAppFileLayersCustomizer, registered in the +// test app's manifest. +describe("App file layers", function () { + var dir = __dirname + "/appFileLayers/"; + + it("serves a map entry from an APK asset under the requested path", function () { + var remapped = require("./appFileLayers/remapped.js"); + expect(remapped.value).toBe("target"); + expect(remapped.filename).toBe(dir + "remapped.js"); + expect(com.tns.AppFiles.isFile(dir + "remapped.js")).toBe(true); + }); + + it("serves files from a customizer's directory layer", function () { + expect(require("./appFileLayers/fromDirectory.js").value).toBe("directory"); + }); + + it("serves ES modules from a customizer's directory layer", function (done) { + import(dir + "fromDirectory.mjs").then(function (ns) { + expect(ns.value).toBe("directory-esm"); + done(); + }, function (e) { + expect(String(e)).toBe(""); + done(); + }); + }); + + it("serves files from a customizer's APK asset layer", function () { + expect(require("./appFileLayers/fromApkBase.js").value).toBe("apk-assets"); + }); + + it("hides a whited-out path that a lower layer has", function () { + expect(com.tns.AppFiles.isFile(dir + "whitedOut.js")).toBe(false); + expect(function () { + require("./appFileLayers/whitedOut.js"); + }).toThrow(); + }); + + it("never layers the runtime's internal files", function () { + expect(com.tns.AppFiles.isFile(__approot + "/internal/ts_helpers.js")).toBe(true); + expect(com.tns.AppFiles.isFile(__approot + "/internal/tests/appFileLayers/remapped.js")).toBe(false); + }); +}); diff --git a/test-app/app/src/main/assets/app/tests/testCodeCache.js b/test-app/app/src/main/assets/app/tests/testCodeCache.js new file mode 100644 index 000000000..a23dbbc37 --- /dev/null +++ b/test-app/app/src/main/assets/app/tests/testCodeCache.js @@ -0,0 +1,185 @@ +// Needs "codeCache": true in the test app's package.json. Every load runs in a +// fresh worker, which has its own module registry, so the probe compiles again +// and consults the cache entry the previous load left behind. +describe("V8 code cache", function () { + var File = java.io.File; + var HEADER_SIZE = 24; + var UNTOUCHED_MTIME = 1000000; + + var filesDir = new File(__approot); + var cacheDir = new File(filesDir.getParentFile(), "code_cache/v8"); + var probeDir = new File(filesDir, "app/codeCacheProbe"); + + // Mirrors the runtime's entry name: FNV-1a 64 of the logical path. + function cacheFileFor(path, isModule) { + var hash = 0xcbf29ce484222325n; + var mask = (1n << 64n) - 1n; + var bytes = new TextEncoder().encode(path); + for (var i = 0; i < bytes.length; i++) { + hash ^= BigInt(bytes[i]); + hash = (hash * 0x100000001b3n) & mask; + } + var name = hash.toString(16).padStart(16, "0") + (isModule ? ".mcache" : ".cache"); + return new File(cacheDir, name); + } + + function writeText(file, text) { + var writer = new java.io.FileWriter(file); + writer.write(text); + writer.close(); + } + + function readBytes(file) { + var raf = new java.io.RandomAccessFile(file, "r"); + var bytes = Array.create("byte", raf.length()); + raf.readFully(bytes); + raf.close(); + return bytes; + } + + function writeByte(file, offset, value) { + var raf = new java.io.RandomAccessFile(file, "rw"); + raf.seek(offset); + raf.write(value); + raf.close(); + } + + function readUint32(bytes, offset) { + return ((bytes[offset] & 255) | + ((bytes[offset + 1] & 255) << 8) | + ((bytes[offset + 2] & 255) << 16) | + ((bytes[offset + 3] & 255) << 24)) >>> 0; + } + + function writeUint32(file, offset, value) { + for (var i = 0; i < 4; i++) { + writeByte(file, offset + i, (value >>> (8 * i)) & 255); + } + } + + function payloadCrc(bytes) { + var crc = new java.util.zip.CRC32(); + crc.update(bytes, HEADER_SIZE, bytes.length - HEADER_SIZE); + return Number(crc.getValue()); + } + + function expectValidEntry(file) { + expect(file.isFile()).toBe(true); + var bytes = readBytes(file); + expect(bytes.length).toBeGreaterThan(HEADER_SIZE); + var magic = String.fromCharCode(bytes[0], bytes[1], bytes[2], bytes[3]); + expect(magic).toBe("NSCC"); + expect(readUint32(bytes, 16)).toBe(bytes.length - HEADER_SIZE); + expect(readUint32(bytes, 20)).toBe(payloadCrc(bytes)); + return bytes; + } + + function deleteTree(file) { + var children = file.listFiles(); + if (children) { + for (var i = 0; i < children.length; i++) { + deleteTree(children[i]); + } + } + file.delete(); + } + + function loadInWorker(probe, callback) { + var worker = new Worker("./codeCacheLoadWorker.js"); + worker.onmessage = function (msg) { + worker.terminate(); + callback(msg.data); + }; + worker.postMessage({ path: probe.path, isModule: probe.isModule }); + } + + [false, true].forEach(function (isModule) { + describe(isModule ? "for ES modules" : "for CommonJS modules", function () { + // Created by the first spec and removed by the last: the specs build + // on each other's cache entry, in order. + var probe; + + function createProbe() { + probeDir.mkdirs(); + var token = "probe-" + Date.now() + "-" + Math.floor(Math.random() * 1e9); + var path = probeDir.getAbsolutePath() + "/" + token + (isModule ? ".mjs" : ".js"); + writeText(new File(path), isModule + ? "export const value = '" + token + "';\n" + : "module.exports.value = '" + token + "';\n"); + probe = { + path: path, + isModule: isModule, + token: token, + cacheFile: cacheFileFor(path, isModule) + }; + } + + function removeProbe() { + new File(probe.path).delete(); + probe.cacheFile.delete(); + probeDir.delete(); + } + + it("writes an entry on the first compile", function (done) { + createProbe(); + loadInWorker(probe, function (result) { + expect(result.error).toBeUndefined(); + expect(result.value).toBe(probe.token); + expectValidEntry(probe.cacheFile); + done(); + }); + }); + + it("consumes a valid entry without rewriting it", function (done) { + expect(probe.cacheFile.setLastModified(UNTOUCHED_MTIME)).toBe(true); + loadInWorker(probe, function (result) { + expect(result.value).toBe(probe.token); + expect(Number(probe.cacheFile.lastModified())).toBe(UNTOUCHED_MTIME); + done(); + }); + }); + + it("rejects an entry whose payload checksum fails and rewrites it", function (done) { + var bytes = readBytes(probe.cacheFile); + var offset = HEADER_SIZE + Math.floor((bytes.length - HEADER_SIZE) / 2); + writeByte(probe.cacheFile, offset, (bytes[offset] & 255) ^ 0xff); + expect(probe.cacheFile.setLastModified(UNTOUCHED_MTIME)).toBe(true); + loadInWorker(probe, function (result) { + expect(result.value).toBe(probe.token); + expect(Number(probe.cacheFile.lastModified())).not.toBe(UNTOUCHED_MTIME); + expectValidEntry(probe.cacheFile); + done(); + }); + }); + + it("rewrites an entry V8 rejects", function (done) { + // Corrupt V8's own header (its magic number) but keep our + // checksum valid, so the entry reaches V8 and V8 rejects it. + var bytes = readBytes(probe.cacheFile); + var original = bytes[HEADER_SIZE] & 255; + bytes[HEADER_SIZE] = ((original ^ 0xff) << 24) >> 24; + writeByte(probe.cacheFile, HEADER_SIZE, original ^ 0xff); + writeUint32(probe.cacheFile, 20, payloadCrc(bytes)); + expect(probe.cacheFile.setLastModified(UNTOUCHED_MTIME)).toBe(true); + loadInWorker(probe, function (result) { + expect(result.value).toBe(probe.token); + expect(Number(probe.cacheFile.lastModified())).not.toBe(UNTOUCHED_MTIME); + var rewritten = expectValidEntry(probe.cacheFile); + expect(rewritten[HEADER_SIZE] & 255).toBe(original); + done(); + }); + }); + + it("tolerates a missing cache directory", function (done) { + deleteTree(cacheDir); + expect(cacheDir.exists()).toBe(false); + loadInWorker(probe, function (result) { + expect(result.value).toBe(probe.token); + expectValidEntry(probe.cacheFile); + removeProbe(); + done(); + }); + }); + }); + }); +}); diff --git a/test-app/app/src/main/assets/app/tests/testLoadCodeFromApk.js b/test-app/app/src/main/assets/app/tests/testLoadCodeFromApk.js new file mode 100644 index 000000000..61a087ca6 --- /dev/null +++ b/test-app/app/src/main/assets/app/tests/testLoadCodeFromApk.js @@ -0,0 +1,158 @@ +// Runs in both modes; the suite is run once as built and once with +// -PloadCodeFromApk (see tools/appfiles-test-mode.init.gradle). +describe("App files", function () { + var File = java.io.File; + var loadsCodeFromApk = com.tns.AppFiles.loadsCodeFromApk(); + var filesRoot = __approot; + var appRoot = filesRoot + "/app"; + + function otherSpelling(path) { + if (path.indexOf("/data/data/") === 0) { + return "/data/user/0/" + path.substring("/data/data/".length); + } + if (path.indexOf("/data/user/0/") === 0) { + return "/data/data/" + path.substring("/data/user/0/".length); + } + return path; + } + + function loadInWorker(path, callback) { + var worker = new Worker("./codeCacheLoadWorker.js"); + worker.onmessage = function (msg) { + worker.terminate(); + callback(msg.data); + }; + worker.postMessage({ path: path, isModule: false }); + } + + describe("under either spelling of the files directory", function () { + var aliasRoot = otherSpelling(filesRoot); + + it("has two spellings to test on this device", function () { + expect(aliasRoot).not.toBe(filesRoot); + }); + + it("finds runtime files", function () { + expect(com.tns.AppFiles.isFile(aliasRoot + "/internal/livesync.js")).toBe(true); + expect(com.tns.AppFiles.isFile(aliasRoot + "/app/tests/appFileLayers/target.js")).toBe(true); + }); + + it("runs a script named through getFilesDir()", function () { + var context = com.tns.NativeScriptApplication.getInstance(); + var previous = global.__onLiveSync; + var calls = 0; + global.__onLiveSync = function () { + calls++; + }; + try { + com.tns.Runtime.getCurrentRuntime().runScript(new File(context.getFilesDir(), "internal/livesync.js")); + com.tns.Runtime.getCurrentRuntime().runScript(new File(aliasRoot, "internal/livesync.js")); + } finally { + global.__onLiveSync = previous; + } + expect(calls).toBe(2); + }); + + it("requires one CommonJS module", function () { + var viaAlias = require(aliasRoot + "/app/tests/appFileLayers/target.js"); + var viaRoot = require(appRoot + "/tests/appFileLayers/target.js"); + expect(viaAlias).toBe(viaRoot); + expect(viaAlias.filename).toBe(appRoot + "/tests/appFileLayers/target.js"); + }); + + it("imports one ES module", function (done) { + Promise.all([ + import(aliasRoot + "/app/tests/loadCodeFromApk/aliasTarget.mjs"), + import(appRoot + "/tests/loadCodeFromApk/aliasTarget.mjs") + ]).then(function (namespaces) { + expect(namespaces[0].value).toBe("alias"); + expect(namespaces[0]).toBe(namespaces[1]); + done(); + }, function (e) { + expect(String(e)).toBe(""); + done(); + }); + }); + }); + + it("extracts what android.alwaysExtract names", function () { + expect(new File(appRoot + "/tests/alwaysExtracted/kept.js").isFile()).toBe(true); + expect(require("./alwaysExtracted/kept.js").value).toBe("kept"); + }); + + if (!loadsCodeFromApk) { + it("extracts code when android.loadCodeFromApk is off", function () { + expect(new File(appRoot + "/mainpage.js").isFile()).toBe(true); + expect(new File(filesRoot + "/internal/ts_helpers.js").isFile()).toBe(true); + expect(new File(filesRoot + "/metadata/treeNodeStream.dat").isFile()).toBe(true); + }); + return; + } + + describe("with android.loadCodeFromApk", function () { + function countFiles(dir, extensions) { + var count = 0; + var children = dir.listFiles(); + for (var i = 0; children && i < children.length; i++) { + var child = children[i]; + if (child.isDirectory()) { + count += countFiles(child, extensions); + } else if (extensions.test(child.getName())) { + count++; + } + } + return count; + } + + it("serves code from the APK without extracting it", function () { + expect(countFiles(new File(appRoot + "/tests"), /\.(js|mjs|cjs|map)$/)).toBe( + // the one alwaysExtract match + 1); + expect(new File(appRoot + "/mainpage.js").exists()).toBe(false); + expect(com.tns.AppFiles.isFile(appRoot + "/mainpage.js")).toBe(true); + expect(require("./appFileLayers/target.js").value).toBe("target"); + }); + + it("still extracts JSON, images and other files", function () { + expect(new File(appRoot + "/package.json").isFile()).toBe(true); + expect(new File(appRoot + "/tests/image.jpg").isFile()).toBe(true); + }); + + it("serves internal files and metadata from the APK, removing extracted copies", function () { + expect(new File(filesRoot + "/internal").exists()).toBe(false); + expect(new File(filesRoot + "/metadata").exists()).toBe(false); + expect(com.tns.AppFiles.isFile(filesRoot + "/internal/ts_helpers.js")).toBe(true); + expect(com.tns.AppFiles.isFile(filesRoot + "/metadata/treeNodeStream.dat")).toBe(true); + }); + + it("loads APK-served code in a worker", function (done) { + loadInWorker(appRoot + "/tests/loadCodeFromApk/liveSyncOverride.js", function (result) { + expect(result.error).toBeUndefined(); + expect(result.value).toBe("apk"); + done(); + }); + }); + + it("lets a file LiveSync writes into files/app win over the APK copy", function (done) { + var target = new File(appRoot + "/tests/loadCodeFromApk/liveSyncOverride.js"); + var runtime = com.tns.Runtime.getCurrentRuntime(); + target.getParentFile().mkdirs(); + runtime.lock(); + try { + var out = new java.io.FileOutputStream(target); + out.write(new java.lang.String("module.exports.value = 'disk';\n").getBytes("UTF-8")); + out.close(); + } finally { + runtime.unlock(); + } + loadInWorker(target.getAbsolutePath(), function (overridden) { + target.delete(); + loadInWorker(target.getAbsolutePath(), function (restored) { + expect(overridden.value).toBe("disk"); + expect(restored.value).toBe("apk"); + done(); + }); + }); + }); + }); +}); diff --git a/test-app/app/src/main/assets/layertest/tests/appFileLayers/fromApkBase.js b/test-app/app/src/main/assets/layertest/tests/appFileLayers/fromApkBase.js new file mode 100644 index 000000000..30529d92f --- /dev/null +++ b/test-app/app/src/main/assets/layertest/tests/appFileLayers/fromApkBase.js @@ -0,0 +1 @@ +module.exports.value = 'apk-assets'; diff --git a/test-app/app/src/main/java/com/tns/AndroidJsV8Inspector.java b/test-app/app/src/main/java/com/tns/AndroidJsV8Inspector.java index fd1ef7329..483064791 100644 --- a/test-app/app/src/main/java/com/tns/AndroidJsV8Inspector.java +++ b/test-app/app/src/main/java/com/tns/AndroidJsV8Inspector.java @@ -129,7 +129,7 @@ public static Pair[] getPageResources() { List> resources = traverseResources(rootFilesDir); @SuppressWarnings("unchecked") - Pair[] result = resources.toArray((Pair[]) Array.newInstance(resources.get(0).getClass(), resources.size())); + Pair[] result = resources.toArray((Pair[]) Array.newInstance(Pair.class, resources.size())); return result; } @@ -143,6 +143,9 @@ private static List> traverseResources(File dir) { File currentDir = directories.poll(); File[] files = currentDir.listFiles(); + if (files == null) { + continue; + } for (File file : files) { if (file.isDirectory()) { directories.add(file); diff --git a/test-app/app/src/main/java/com/tns/RuntimeHelper.java b/test-app/app/src/main/java/com/tns/RuntimeHelper.java index fa1542966..a35e3c970 100644 --- a/test-app/app/src/main/java/com/tns/RuntimeHelper.java +++ b/test-app/app/src/main/java/com/tns/RuntimeHelper.java @@ -83,6 +83,7 @@ public static Runtime initRuntime(Context context) { Thread.setDefaultUncaughtExceptionHandler(exHandler); DefaultExtractPolicy extractPolicy = new DefaultExtractPolicy(logger); + AppFiles.init(context, extractPolicy.shouldExtract(context)); boolean skipAssetExtraction = Util.runPlugin(logger, context); String appName = context.getPackageName(); @@ -108,9 +109,13 @@ public static Runtime initRuntime(Context context) { // will force deletion of previously extracted files in app/files directories // see https://github.com/NativeScript/NativeScript/issues/4137 for reference boolean removePreviouslyInstalledAssets = true; - aE.extractAssets(context, "app", outputDir, extractPolicy, removePreviouslyInstalledAssets); - aE.extractAssets(context, "internal", outputDir, extractPolicy, removePreviouslyInstalledAssets); - aE.extractAssets(context, "metadata", outputDir, extractPolicy, false); + if (AppFiles.loadsCodeFromApk() && removePreviousVersionFiles(context, extractPolicy)) { + aE.extractAssets(context, "app", outputDir, extractPolicy, false, true); + } else { + aE.extractAssets(context, "app", outputDir, extractPolicy, removePreviouslyInstalledAssets, false); + aE.extractAssets(context, "internal", outputDir, extractPolicy, removePreviouslyInstalledAssets, false); + aE.extractAssets(context, "metadata", outputDir, extractPolicy, false, false); + } extractPolicy.setAssetsThumb(context); } finally { @@ -205,7 +210,7 @@ public static Runtime initRuntime(Context context) { // runtime.runScript(new File(appDir, "internal/ts_helpers.js")); File javaClassesModule = new File(appDir, "app/tns-java-classes.js"); - if (javaClassesModule.exists()) { + if (AppFiles.isFile(javaClassesModule.getAbsolutePath())) { runtime.runModule(javaClassesModule); } @@ -237,6 +242,36 @@ public static Runtime initRuntime(Context context) { } } + /* + * Extracted files are searched before the APK, so on an update every copy + * the previous version left must go, or it would shadow the new APK. When + * that fails, everything is extracted over it instead, as without + * loadCodeFromApk. + */ + private static boolean removePreviousVersionFiles(Context context, ExtractPolicy extractPolicy) { + if (!extractPolicy.shouldExtract(context)) { + return true; + } + File filesDir = context.getFilesDir(); + boolean removed = deleteRecursively(new File(filesDir, "app")); + removed = deleteRecursively(new File(filesDir, "internal")) && removed; + removed = deleteRecursively(new File(filesDir, "metadata")) && removed; + if (!removed) { + Log.w(logTag, "Could not remove the files of the previous app version; extracting all assets, code included, so none of them is served in place of the APK's copy."); + } + return removed; + } + + private static boolean deleteRecursively(File file) { + File[] children = file.listFiles(); + if (children != null) { + for (File child : children) { + deleteRecursively(child); + } + } + return file.delete() || !file.exists(); + } + private static void waitForLiveSync(Context context) { boolean needToWait = false; diff --git a/test-app/app/src/main/java/com/tns/tests/TestAppFileLayersCustomizer.java b/test-app/app/src/main/java/com/tns/tests/TestAppFileLayersCustomizer.java new file mode 100644 index 000000000..bef867369 --- /dev/null +++ b/test-app/app/src/main/java/com/tns/tests/TestAppFileLayersCustomizer.java @@ -0,0 +1,49 @@ +package com.tns.tests; + +import com.tns.AppFileLayer; +import com.tns.AppFileLayers; +import com.tns.AppFileLayersCustomizer; + +import java.io.File; +import java.io.FileOutputStream; +import java.io.IOException; +import java.nio.charset.StandardCharsets; +import java.util.HashMap; +import java.util.Map; + +// Backs tests/testAppFileLayers.js; the layers answer only paths under tests/appFileLayers/. +public class TestAppFileLayersCustomizer implements AppFileLayersCustomizer { + static final String MAP_LAYER = "test-map"; + static final String DIRECTORY_LAYER = "test-directory"; + static final String ASSETS_LAYER = "test-assets"; + + @Override + public int priority() { + return 0; + } + + @Override + public void customize(AppFileLayers.Builder builder) { + File dir = new File(builder.context().getNoBackupFilesDir(), "appFileLayersTest"); + write(new File(dir, "tests/appFileLayers/fromDirectory.js"), "module.exports.value = 'directory';\n"); + write(new File(dir, "tests/appFileLayers/fromDirectory.mjs"), "export const value = 'directory-esm';\n"); + write(new File(dir, "tests/appFileLayers/whitedOut.js"), "module.exports.value = 'should be hidden';\n"); + + Map entries = new HashMap<>(); + entries.put("tests/appFileLayers/remapped.js", AppFileLayer.Entry.asset("app/tests/appFileLayers/target.js")); + entries.put("tests/appFileLayers/whitedOut.js", AppFileLayer.Entry.whiteout()); + + builder.insertBelow(AppFileLayers.EXTRACTED, AppFileLayer.map(MAP_LAYER, entries)); + builder.insertBelow(MAP_LAYER, AppFileLayer.directory(DIRECTORY_LAYER, dir)); + builder.insertBelow(DIRECTORY_LAYER, AppFileLayer.apkAssets(ASSETS_LAYER, "layertest")); + } + + private static void write(File file, String text) { + file.getParentFile().mkdirs(); + try (FileOutputStream out = new FileOutputStream(file)) { + out.write(text.getBytes(StandardCharsets.UTF_8)); + } catch (IOException e) { + throw new RuntimeException(e); + } + } +} diff --git a/test-app/runtests.gradle b/test-app/runtests.gradle index d8944745c..bdad9d40c 100644 --- a/test-app/runtests.gradle +++ b/test-app/runtests.gradle @@ -75,6 +75,14 @@ def getBuildArguments = { -> arguments.add("-PuseKotlin=true") + arguments += ["-I", "$rootDir/tools/appfiles-test-mode.init.gradle"] + if (project.hasProperty("loadCodeFromApk")) { + arguments.add("-PloadCodeFromApk") + } + if (project.hasProperty("storeCodeUncompressed")) { + arguments.add("-PstoreCodeUncompressed") + } + return arguments } diff --git a/test-app/runtime/CMakeLists.txt b/test-app/runtime/CMakeLists.txt index fb87868c4..fee0427c7 100644 --- a/test-app/runtime/CMakeLists.txt +++ b/test-app/runtime/CMakeLists.txt @@ -174,6 +174,7 @@ add_library( SHARED # Runtime source + src/main/cpp/AppFiles.cpp src/main/cpp/ArgConverter.cpp src/main/cpp/ArrayBufferHelper.cpp src/main/cpp/ArrayElementAccessor.cpp @@ -191,7 +192,6 @@ add_library( src/main/cpp/ExtendFileIdentifier.cpp src/main/cpp/FrameCallbacks.cpp src/main/cpp/FieldAccessor.cpp - src/main/cpp/File.cpp src/main/cpp/Interop.cpp src/main/cpp/IsolateTracked.cpp src/main/cpp/JEnv.cpp @@ -225,6 +225,7 @@ add_library( src/main/cpp/ReadWriteLock.cpp src/main/cpp/Runtime.cpp src/main/cpp/RuntimeState.cpp + src/main/cpp/ScriptCache.cpp src/main/cpp/SimpleAllocator.cpp src/main/cpp/SimpleProfiler.cpp src/main/cpp/StructuredClone.cpp @@ -237,6 +238,7 @@ add_library( src/main/cpp/WorkerEvents.cpp src/main/cpp/WorkerWrapper.cpp src/main/cpp/Timers.cpp + src/main/cpp/com_tns_AppFiles.cpp src/main/cpp/com_tns_AssetExtractor.cpp src/main/cpp/com_tns_Runtime.cpp src/main/cpp/console/Console.cpp diff --git a/test-app/runtime/consumer-rules.pro b/test-app/runtime/consumer-rules.pro index 329ee7266..d5dfa2902 100644 --- a/test-app/runtime/consumer-rules.pro +++ b/test-app/runtime/consumer-rules.pro @@ -8,4 +8,6 @@ -keep class com.tns.internal.** { *; } # Preserve annotation metadata so reflection sees them --keepattributes RuntimeVisibleAnnotations \ No newline at end of file +-keepattributes RuntimeVisibleAnnotations +# Customizers are only named in the manifest and created by reflection +-keep class * implements com.tns.AppFileLayersCustomizer { (); } diff --git a/test-app/runtime/src/main/cpp/AppFiles.cpp b/test-app/runtime/src/main/cpp/AppFiles.cpp new file mode 100644 index 000000000..5cddc6e58 --- /dev/null +++ b/test-app/runtime/src/main/cpp/AppFiles.cpp @@ -0,0 +1,658 @@ +#include "AppFiles.h" + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace tns { +namespace AppFiles { + +namespace { + +enum class Lookup { + Miss, + Hit, + // Stop: not found, whatever the sources below hold. + Whiteout, +}; + +/* + * One place app files can come from. `relative` is relative to the files + * root, e.g. "app/main.js" or "metadata/treeNodeStream.dat". Sources are + * immutable after Init, apart from internal caches they guard themselves. + * Layers from customizers only answer paths under app/. + */ +class Source { +public: + virtual ~Source() = default; + virtual Lookup Find(const std::string& relative) const = 0; + // Only called after Find hit; null with errno set when the open fails. + virtual std::unique_ptr Open(const std::string& relative) const = 0; + virtual bool RealFilePath(const std::string& relative, std::string& realPath) const { + return false; + } +}; + +// Which part of the layered roots a source answers for. +enum class Serves { + Everything, + ApkServedOnly, + AppOnly, +}; + +bool InScope(Serves serves, const std::string& relative) { + switch (serves) { + case Serves::Everything: + return true; + case Serves::ApkServedOnly: + return IsApkServed(relative); + case Serves::AppOnly: + return relative.compare(0, 4, "app/") == 0; + } + return false; +} + +class FileReader final : public Reader { +public: + FileReader(int fd, size_t size) : m_fd(fd), m_size(size) {} + ~FileReader() override { close(m_fd); } + + size_t size() const override { return m_size; } + + size_t read(void* dst, size_t n) override { + auto* out = static_cast(dst); + size_t total = 0; + while (total < n) { + ssize_t r = ::read(m_fd, out + total, n - total); + if (r < 0 && errno == EINTR) { + continue; + } + if (r <= 0) { + break; + } + total += static_cast(r); + } + return total; + } + +private: + int m_fd; + size_t m_size; +}; + +bool IsRegularFile(const std::string& path) { + struct stat st; + return stat(path.c_str(), &st) == 0 && S_ISREG(st.st_mode); +} + +std::unique_ptr OpenRegularFile(const std::string& path) { + int fd = open(path.c_str(), O_RDONLY | O_CLOEXEC); + if (fd < 0) { + return nullptr; + } + struct stat st; + int error = 0; + if (fstat(fd, &st) != 0) { + error = errno; + } else if (!S_ISREG(st.st_mode)) { + error = S_ISDIR(st.st_mode) ? EISDIR : EINVAL; + } + if (error != 0) { + close(fd); + errno = error; + return nullptr; + } + return std::unique_ptr(new FileReader(fd, static_cast(st.st_size))); +} + +class AssetReader final : public Reader { +public: + explicit AssetReader(AAsset* asset) : m_asset(asset) {} + ~AssetReader() override { AAsset_close(m_asset); } + + size_t size() const override { return static_cast(AAsset_getLength64(m_asset)); } + + size_t read(void* dst, size_t n) override { + auto* out = static_cast(dst); + size_t total = 0; + while (total < n) { + int r = AAsset_read(m_asset, out + total, n - total); + if (r <= 0) { + break; + } + total += static_cast(r); + } + return total; + } + +private: + AAsset* m_asset; +}; + +/* + * Files under `root`. A built-in source mirrors the files root; a customizer's + * directory is laid out like the app root. + */ +class DirectorySource final : public Source { +public: + DirectorySource(std::string root, Serves serves) + : m_root(std::move(root)), m_serves(serves) {} + + Lookup Find(const std::string& relative) const override { + return InScope(m_serves, relative) && IsRegularFile(PathFor(relative)) ? Lookup::Hit + : Lookup::Miss; + } + + std::unique_ptr Open(const std::string& relative) const override { + return OpenRegularFile(PathFor(relative)); + } + + bool RealFilePath(const std::string& relative, std::string& realPath) const override { + realPath = PathFor(relative); + return true; + } + +private: + std::string PathFor(const std::string& relative) const { + return m_root + "/" + (m_serves == Serves::AppOnly ? relative.substr(4) : relative); + } + + std::string m_root; + Serves m_serves; +}; + +/* + * Which APK assets exist. One AAssetManager_openDir listing per directory, + * taken on first use; AAssets themselves are never shared, every open gets + * its own. + */ +class AssetIndex { +public: + explicit AssetIndex(AAssetManager* assets) : m_assets(assets) {} + + AAssetManager* assets() const { return m_assets; } + + bool Exists(const std::string& assetPath) const { + size_t slash = assetPath.find_last_of('/'); + std::string dir = slash == std::string::npos ? std::string() : assetPath.substr(0, slash); + std::string name = slash == std::string::npos ? assetPath : assetPath.substr(slash + 1); + + std::lock_guard lock(m_mutex); + auto it = m_listings.find(dir); + if (it == m_listings.end()) { + std::unordered_set names; + if (AAssetDir* listing = AAssetManager_openDir(m_assets, dir.c_str())) { + while (const char* entry = AAssetDir_getNextFileName(listing)) { + names.insert(entry); + } + AAssetDir_close(listing); + } + it = m_listings.emplace(std::move(dir), std::move(names)).first; + } + return it->second.count(name) != 0; + } + + std::unique_ptr Open(const std::string& assetPath) const { + AAsset* asset = AAssetManager_open(m_assets, assetPath.c_str(), AASSET_MODE_STREAMING); + if (asset == nullptr) { + errno = ENOENT; + return nullptr; + } + return std::unique_ptr(new AssetReader(asset)); + } + +private: + AAssetManager* m_assets; + mutable std::mutex m_mutex; + // Directory -> the regular files directly in it. + mutable std::unordered_map> m_listings; +}; + +/* + * APK assets. The built-in source mirrors the files root ("app/main.js" is + * the asset "app/main.js"); a customizer's lays out `base` like the app root. + */ +class ApkSource final : public Source { +public: + ApkSource(const AssetIndex& index, Serves serves, std::string base = std::string()) + : m_index(index), m_serves(serves), m_base(std::move(base)) {} + + Lookup Find(const std::string& relative) const override { + return InScope(m_serves, relative) && m_index.Exists(AssetFor(relative)) ? Lookup::Hit + : Lookup::Miss; + } + + std::unique_ptr Open(const std::string& relative) const override { + return m_index.Open(AssetFor(relative)); + } + +private: + std::string AssetFor(const std::string& relative) const { + return m_serves == Serves::AppOnly ? m_base + "/" + relative.substr(4) : relative; + } + + const AssetIndex& m_index; + Serves m_serves; + std::string m_base; +}; + +// Exactly the listed app-relative paths, each a file, an asset or a whiteout. +class MapSource final : public Source { +public: + MapSource(const AssetIndex* index, std::unordered_map entries) + : m_index(index), m_entries(std::move(entries)) {} + + Lookup Find(const std::string& relative) const override { + const LayerEntry* entry = EntryFor(relative); + if (entry == nullptr) { + return Lookup::Miss; + } + switch (entry->kind) { + case LayerEntry::Kind::File: + return IsRegularFile(entry->target) ? Lookup::Hit : Lookup::Miss; + case LayerEntry::Kind::Asset: + return m_index != nullptr && m_index->Exists(entry->target) ? Lookup::Hit + : Lookup::Miss; + case LayerEntry::Kind::Whiteout: + return Lookup::Whiteout; + } + return Lookup::Miss; + } + + std::unique_ptr Open(const std::string& relative) const override { + const LayerEntry* entry = EntryFor(relative); + if (entry != nullptr && entry->kind == LayerEntry::Kind::File) { + return OpenRegularFile(entry->target); + } + if (entry != nullptr && entry->kind == LayerEntry::Kind::Asset && m_index != nullptr) { + return m_index->Open(entry->target); + } + errno = ENOENT; + return nullptr; + } + + bool RealFilePath(const std::string& relative, std::string& realPath) const override { + const LayerEntry* entry = EntryFor(relative); + if (entry == nullptr || entry->kind != LayerEntry::Kind::File) { + return false; + } + realPath = entry->target; + return true; + } + +private: + const LayerEntry* EntryFor(const std::string& relative) const { + if (relative.compare(0, 4, "app/") != 0) { + return nullptr; + } + auto it = m_entries.find(relative.substr(4)); + return it == m_entries.end() ? nullptr : &it->second; + } + + const AssetIndex* m_index; + std::unordered_map m_entries; +}; + +struct State { + std::string files; + std::string app; + std::string appPrefix; + std::string internal; + std::string metadata; + // Spellings of `files` other than itself. + std::vector aliases; + bool loadCodeFromApk = false; + std::vector alwaysExtract; + std::unique_ptr assetIndex; + // Searched in order; the first source that has a file serves it. + std::vector> sources; +}; + +State s_state; +std::once_flag s_initOnce; + +// Collapses "//", "." and ".." in an absolute path; ".." never climbs above "/". +std::string CollapseDotSegments(const std::string& path) { + std::vector segments; + size_t i = 0; + while (i <= path.size()) { + size_t j = path.find('/', i); + if (j == std::string::npos) { + j = path.size(); + } + std::string segment = path.substr(i, j - i); + if (segment == "..") { + if (!segments.empty()) { + segments.pop_back(); + } + } else if (!segment.empty() && segment != ".") { + segments.push_back(std::move(segment)); + } + i = j + 1; + } + std::string result; + for (const auto& segment : segments) { + result += "/"; + result += segment; + } + return result.empty() ? "/" : result; +} + +bool HasDirPrefix(const std::string& path, const std::string& dir) { + return path.size() > dir.size() && path.compare(0, dir.size(), dir) == 0 && + path[dir.size()] == '/'; +} + +// /data/data/ and /data/user/0/ name the same directory. +std::string SwappedDataSpelling(const std::string& path) { + static const std::string kDataData = "/data/data/"; + static const std::string kDataUser0 = "/data/user/0/"; + if (path.compare(0, kDataData.size(), kDataData) == 0) { + return kDataUser0 + path.substr(kDataData.size()); + } + if (path.compare(0, kDataUser0.size(), kDataUser0) == 0) { + return kDataData + path.substr(kDataUser0.size()); + } + return std::string(); +} + +void AddAlias(const std::string& alias) { + if (alias.empty() || alias[0] != '/') { + return; + } + std::string collapsed = CollapseDotSegments(alias); + if (collapsed == s_state.files) { + return; + } + for (const auto& existing : s_state.aliases) { + if (existing == collapsed) { + return; + } + } + s_state.aliases.push_back(std::move(collapsed)); +} + +bool HasPrefix(const std::string& s, const char* prefix) { + return s.compare(0, strlen(prefix), prefix) == 0; +} + +/* + * Glob match where "*" and "?" stay within one path segment and "**" spans + * segments; a "**" segment followed by a slash also matches zero directories. + */ +bool GlobMatch(const char* pattern, const char* path) { + while (*pattern != '\0') { + if (pattern[0] == '*' && pattern[1] == '*') { + pattern += 2; + if (*pattern == '/') { + ++pattern; + for (const char* p = path;; ++p) { + if ((p == path || p[-1] == '/') && GlobMatch(pattern, p)) { + return true; + } + if (*p == '\0') { + return false; + } + } + } + for (const char* p = path;; ++p) { + if (GlobMatch(pattern, p)) { + return true; + } + if (*p == '\0') { + return false; + } + } + } + if (*pattern == '*') { + ++pattern; + for (const char* p = path;; ++p) { + if (GlobMatch(pattern, p)) { + return true; + } + if (*p == '\0' || *p == '/') { + return false; + } + } + } + if (*path == '\0') { + return false; + } + if (*pattern == '?' ? *path == '/' : *pattern != *path) { + return false; + } + ++pattern; + ++path; + } + return *path == '\0'; +} + +bool IsCodeExtension(const std::string& relative) { + size_t slash = relative.find_last_of('/'); + size_t dot = relative.find_last_of('.'); + if (dot == std::string::npos || (slash != std::string::npos && dot < slash)) { + return false; + } + const char* ext = relative.c_str() + dot + 1; + return strcmp(ext, "js") == 0 || strcmp(ext, "mjs") == 0 || strcmp(ext, "cjs") == 0 || + strcmp(ext, "map") == 0; +} + +// The path relative to the files root when `path` names something under one +// of the layered roots (app, internal, metadata). +bool LayeredRelativePath(const std::string& path, std::string& relative) { + std::string logical; + if (!ToLogical(path, logical) || !HasDirPrefix(logical, s_state.files)) { + return false; + } + std::string rest = logical.substr(s_state.files.size() + 1); + if (HasPrefix(rest, "app/") || HasPrefix(rest, "internal/") || HasPrefix(rest, "metadata/")) { + relative = std::move(rest); + return true; + } + return false; +} + +// The first source that has `relative`, or null. +const Source* FindSource(const std::string& relative) { + for (const auto& source : s_state.sources) { + switch (source->Find(relative)) { + case Lookup::Hit: + return source.get(); + case Lookup::Whiteout: + return nullptr; + case Lookup::Miss: + break; + } + } + return nullptr; +} + +// Disk first: what LiveSync or the app itself writes into the files +// directory wins over the APK's copy. +std::vector DefaultLayers() { + std::vector layers(2); + layers[0].kind = Layer::Kind::Extracted; + layers[1].kind = Layer::Kind::Apk; + return layers; +} + +void AddSource(const Layer& layer) { + const AssetIndex* index = s_state.assetIndex.get(); + switch (layer.kind) { + case Layer::Kind::Extracted: + s_state.sources.emplace_back(new DirectorySource(s_state.files, Serves::Everything)); + break; + case Layer::Kind::Apk: + if (s_state.loadCodeFromApk) { + s_state.sources.emplace_back(new ApkSource(*index, Serves::ApkServedOnly)); + } + break; + case Layer::Kind::Directory: + s_state.sources.emplace_back(new DirectorySource(layer.root, Serves::AppOnly)); + break; + case Layer::Kind::ApkAssets: + if (index != nullptr) { + s_state.sources.emplace_back(new ApkSource(*index, Serves::AppOnly, layer.root)); + } + break; + case Layer::Kind::Map: { + std::unordered_map entries; + for (const auto& entry : layer.entries) { + entries[entry.first] = entry.second; + } + s_state.sources.emplace_back(new MapSource(index, std::move(entries))); + break; + } + } +} + +} // namespace + +void Init(const Config& config) { + std::call_once(s_initOnce, [&config]() { + const std::string& filesRoot = config.filesRoot; + s_state.files = filesRoot.empty() || filesRoot[0] != '/' + ? filesRoot + : CollapseDotSegments(filesRoot); + s_state.app = s_state.files + "/app"; + s_state.appPrefix = s_state.app + "/"; + s_state.internal = s_state.files + "/internal"; + s_state.metadata = s_state.files + "/metadata"; + + AddAlias(SwappedDataSpelling(s_state.files)); + for (const auto& alias : config.aliases) { + AddAlias(alias); + AddAlias(SwappedDataSpelling(alias)); + } + + s_state.loadCodeFromApk = config.loadCodeFromApk && config.assets != nullptr; + s_state.alwaysExtract = config.alwaysExtract; + if (config.assets != nullptr) { + s_state.assetIndex.reset(new AssetIndex(config.assets)); + } + + const std::vector& layers = config.layers.empty() ? DefaultLayers() : config.layers; + for (const auto& layer : layers) { + AddSource(layer); + } + }); +} + +void Init(const std::string& filesRoot) { + Config config; + config.filesRoot = filesRoot; + Init(config); +} + +const std::string& FilesRoot() { + return s_state.files; +} + +const std::string& AppRoot() { + return s_state.app; +} + +const std::string& InternalRoot() { + return s_state.internal; +} + +const std::string& MetadataRoot() { + return s_state.metadata; +} + +const std::string& AppRootPrefix() { + return s_state.appPrefix; +} + +bool LoadsCodeFromApk() { + return s_state.loadCodeFromApk; +} + +bool IsApkServed(const std::string& relative) { + if (!s_state.loadCodeFromApk) { + return false; + } + if (HasPrefix(relative, "internal/") || HasPrefix(relative, "metadata/")) { + return true; + } + if (!HasPrefix(relative, "app/") || !IsCodeExtension(relative)) { + return false; + } + const char* appRelative = relative.c_str() + strlen("app/"); + for (const auto& glob : s_state.alwaysExtract) { + if (GlobMatch(glob.c_str(), appRelative)) { + return false; + } + } + return true; +} + +bool ToLogical(const std::string& path, std::string& logical) { + if (s_state.files.empty() || path.empty() || path[0] != '/') { + return false; + } + std::string collapsed = CollapseDotSegments(path); + if (collapsed == s_state.files || HasDirPrefix(collapsed, s_state.files)) { + logical = std::move(collapsed); + return true; + } + for (const auto& alias : s_state.aliases) { + if (collapsed == alias || HasDirPrefix(collapsed, alias)) { + logical = s_state.files + collapsed.substr(alias.size()); + return true; + } + } + return false; +} + +bool IsFile(const std::string& path) { + std::string relative; + if (!LayeredRelativePath(path, relative)) { + return IsRegularFile(path); + } + return FindSource(relative) != nullptr; +} + +std::unique_ptr Open(const std::string& path) { + std::string relative; + if (!LayeredRelativePath(path, relative)) { + return OpenRegularFile(path); + } + const Source* source = FindSource(relative); + if (source == nullptr) { + errno = ENOENT; + return nullptr; + } + return source->Open(relative); +} + +bool ReadText(const std::string& path, std::string& out) { + auto reader = Open(path); + if (reader == nullptr) { + out.clear(); + return false; + } + out.resize(reader->size()); + out.resize(reader->read(&out[0], out.size())); + return true; +} + +bool RealFilePath(const std::string& path, std::string& realPath) { + std::string relative; + if (!LayeredRelativePath(path, relative)) { + realPath = path; + return IsRegularFile(path); + } + const Source* source = FindSource(relative); + return source != nullptr && source->RealFilePath(relative, realPath); +} + +} // namespace AppFiles +} // namespace tns diff --git a/test-app/runtime/src/main/cpp/AppFiles.h b/test-app/runtime/src/main/cpp/AppFiles.h new file mode 100644 index 000000000..d619a0af1 --- /dev/null +++ b/test-app/runtime/src/main/cpp/AppFiles.h @@ -0,0 +1,129 @@ +#ifndef APPFILES_H_ +#define APPFILES_H_ + +#include +#include +#include +#include +#include + +struct AAssetManager; + +namespace tns { +namespace AppFiles { + +// What a map layer answers for one app-relative path. +struct LayerEntry { + enum class Kind { + File, + Asset, + Whiteout, + }; + Kind kind = Kind::Whiteout; + // An absolute file path or an asset path. + std::string target; +}; + +struct Layer { + enum class Kind { + // The files directory, laid out like the files root. + Extracted, + // The APK-served set, straight from the APK. + Apk, + // From customizers; these answer only paths under app/. + Directory, + ApkAssets, + Map, + }; + Kind kind = Kind::Extracted; + // Directory: a directory laid out like the app root. ApkAssets: an asset + // directory laid out like the app root. + std::string root; + // Map: app-relative path -> entry. + std::vector> entries; +}; + +struct Config { + // The canonical files directory: the logical root V8 sees. + std::string filesRoot; + // Other spellings of filesRoot. + std::vector aliases; + // The APK's assets; needed by loadCodeFromApk and by asset layers. + AAssetManager* assets = nullptr; + bool loadCodeFromApk = false; + // Globs relative to the app root that are extracted even when they are code. + std::vector alwaysExtract; + // Searched top first. Empty means the built-in order: the extracted files, + // then the APK. An Apk layer is inert without loadCodeFromApk. + std::vector layers; +}; + +/* + * Only the first call takes effect, so every runtime may call it; the roots + * below are empty until then. Must run before any read and before assets are + * extracted, since the extractor consults IsApkServed. + */ +void Init(const Config& config); + +// Files directory only, every app file read from disk. +void Init(const std::string& filesRoot); + +// The logical "", "/app", "/internal" and "/metadata", +// with no trailing slash. +const std::string& FilesRoot(); +const std::string& AppRoot(); +const std::string& InternalRoot(); +const std::string& MetadataRoot(); + +// "/app/", for prefix checks and joining relative names. +const std::string& AppRootPrefix(); + +bool LoadsCodeFromApk(); + +/* + * The one rule for which files are served from the APK instead of being + * extracted: with loadCodeFromApk, .js/.mjs/.cjs/.map under app/ (minus the + * alwaysExtract globs) and everything under internal/ and metadata/. + * `relative` is relative to the files root, e.g. "app/main.js". + */ +bool IsApkServed(const std::string& relative); + +/* + * True when `path` names something inside the files directory under any of + * its spellings. `logical` is then that path with "//", "." and ".." + * collapsed and the directory spelled as FilesRoot(). + */ +bool ToLogical(const std::string& path, std::string& logical); + +/* + * Reads of app files. A path under the app, internal or metadata root is + * resolved through the app file sources; anything else reads the real file + * system as-is. Safe to call from any thread. + */ +bool IsFile(const std::string& path); + +// False when the file cannot be opened; a short read truncates `out`. +bool ReadText(const std::string& path, std::string& out); + +class Reader { +public: + virtual ~Reader() = default; + virtual size_t size() const = 0; + // Fills `dst` with up to `n` bytes; fewer only at end of file or on error. + virtual size_t read(void* dst, size_t n) = 0; +}; + +// Null when the file cannot be opened, with errno describing why. +// A Reader belongs to one thread at a time. +std::unique_ptr Open(const std::string& path); + +/* + * The real file that serves `path`, for consumers that need one (dlopen). + * False when `path` is served from the APK or does not exist. + */ +bool RealFilePath(const std::string& path, std::string& realPath); + +} // namespace AppFiles +} // namespace tns + +#endif /* APPFILES_H_ */ diff --git a/test-app/runtime/src/main/cpp/AssetExtractor.cpp b/test-app/runtime/src/main/cpp/AssetExtractor.cpp index a4460a04b..61d32cc62 100644 --- a/test-app/runtime/src/main/cpp/AssetExtractor.cpp +++ b/test-app/runtime/src/main/cpp/AssetExtractor.cpp @@ -10,11 +10,13 @@ #include #include #include "AssetExtractor.h" +#include "AppFiles.h" using namespace tns; -void AssetExtractor::ExtractAssets(JNIEnv* env, jobject obj, jstring apk, jstring input, jstring outputDir, jboolean _forceOverwrite) { +void AssetExtractor::ExtractAssets(JNIEnv* env, jobject obj, jstring apk, jstring input, jstring outputDir, jboolean _forceOverwrite, jboolean _skipApkServed) { auto forceOverwrite = JNI_TRUE == _forceOverwrite; + auto skipApkServed = JNI_TRUE == _skipApkServed; auto strApk = jstringToString(env, apk); auto baseDir = jstringToString(env, outputDir); @@ -40,6 +42,9 @@ void AssetExtractor::ExtractAssets(JNIEnv* env, jobject obj, jstring apk, jstrin zip_stat_index(z, i, ZIP_STAT_MTIME, &sb); if (strstr(sb.name, prfx) == sb.name) { auto name = sb.name + prefixLen; // strlen("assets/") == 7 + if (skipApkServed && AppFiles::IsApkServed(name)) { + continue; + } std::string assetFullname(baseDir); assetFullname.append(name); diff --git a/test-app/runtime/src/main/cpp/AssetExtractor.h b/test-app/runtime/src/main/cpp/AssetExtractor.h index fe96a728f..1025b625e 100644 --- a/test-app/runtime/src/main/cpp/AssetExtractor.h +++ b/test-app/runtime/src/main/cpp/AssetExtractor.h @@ -6,7 +6,7 @@ namespace tns { class AssetExtractor { public: - static void ExtractAssets(JNIEnv* env, jobject obj, jstring apk, jstring inputDir, jstring outputDir, jboolean _forceOverwrite); + static void ExtractAssets(JNIEnv* env, jobject obj, jstring apk, jstring inputDir, jstring outputDir, jboolean _forceOverwrite, jboolean skipApkServed); private: static std::string jstringToString(JNIEnv* env, jstring value); diff --git a/test-app/runtime/src/main/cpp/CallbackHandlers.cpp b/test-app/runtime/src/main/cpp/CallbackHandlers.cpp index c4d1e7a2f..886b5da0f 100644 --- a/test-app/runtime/src/main/cpp/CallbackHandlers.cpp +++ b/test-app/runtime/src/main/cpp/CallbackHandlers.cpp @@ -1,4 +1,5 @@ #include "CallbackHandlers.h" +#include "AppFiles.h" #include "NativeScriptAssert.h" #include "MetadataNode.h" #include "Util.h" @@ -1514,7 +1515,7 @@ void CallbackHandlers::NewThreadCallback(const v8::FunctionCallbackInfo= 2 && workerPath[1] == '/' ? workerPath.substr(2) : workerPath.substr(1); - resolvedPath = Constants::APP_ROOT_FOLDER_PATH + tail; + resolvedPath = AppFiles::AppRootPrefix() + tail; } /* @@ -1523,7 +1524,7 @@ void CallbackHandlers::NewThreadCallback(const v8::FunctionCallbackInfo() : StackTrace::CurrentStackTrace(isolate, 1, StackTrace::kScriptName); @@ -1558,13 +1559,13 @@ void CallbackHandlers::NewThreadCallback(const v8::FunctionCallbackInfo -#include -#include - -using namespace std; - -namespace tns { - -string File::ReadText(const string& filePath) { - bool ok; - return ReadText(filePath, ok); -} - -string File::ReadText(const string& filePath, bool& ok) { - ok = false; - - FILE* file = fopen(filePath.c_str(), READ_BINARY); - if (file == nullptr) { - // A path that never existed, or one deleted between a caller's stat and - // this open. Callers surface their own error; reading on regardless - // aborts the process on a null FILE*. - DEBUG_WRITE_FORCE("File::ReadText: cannot open %s", filePath.c_str()); - return string(); - } - - fseek(file, 0, SEEK_END); - auto size = ftell(file); - if (size < 0) { - DEBUG_WRITE_FORCE("File::ReadText: cannot size %s", filePath.c_str()); - fclose(file); - return string(); - } - rewind(file); - - // Read straight into the string that is returned: main and worker runtimes - // read modules concurrently from their own threads, so there is no scratch - // buffer to share, and the caller needs a string either way. - string content(static_cast(size), '\0'); - auto bytesRead = fread(content.data(), 1, static_cast(size), file); - fclose(file); - - // A short read is a truncated file, not a failure to open; `size` is only - // what the directory entry claimed. - content.resize(bytesRead); - - ok = true; - return content; -} - -void* File::ReadBinary(const string& filePath, int& length) { - length = 0; - - auto file = fopen(filePath.c_str(), READ_BINARY); - if (!file) { - return nullptr; - } - - fseek(file, 0, SEEK_END); - length = ftell(file); - rewind(file); - - uint8_t* data = new uint8_t[length]; - fread(data, sizeof(uint8_t), length, file); - fclose(file); - - return data; -} - -bool File::WriteBinary(const string& filePath, const void* data, int length) { - auto file = fopen(filePath.c_str(), WRITE_BINARY); - if (!file) { - return false; - } - - auto writtenBytes = fwrite(data, sizeof(uint8_t), length, file); - fclose(file); - - return writtenBytes == length; -} - -const char* File::WRITE_BINARY = "wb"; -const char* File::READ_BINARY = "rb"; -} diff --git a/test-app/runtime/src/main/cpp/File.h b/test-app/runtime/src/main/cpp/File.h deleted file mode 100644 index 0eb35acaa..000000000 --- a/test-app/runtime/src/main/cpp/File.h +++ /dev/null @@ -1,30 +0,0 @@ -/* - * File.h - * - * Created on: Jun 24, 2015 - * Author: gatanasov - */ - -#ifndef JNI_FILE_H_ -#define JNI_FILE_H_ - -#include - -namespace tns { -class File { - public: - static std::string ReadText(const std::string& filePath); - /* - * `ok` distinguishes a file that could not be opened from one that is - * genuinely empty — the plain overload renders both as "". - */ - static std::string ReadText(const std::string& filePath, bool& ok); - static bool WriteBinary(const std::string& filePath, const void* inData, int length); - static void* ReadBinary(const std::string& filePath, int& length); - private: - static const char* WRITE_BINARY; - static const char* READ_BINARY; -}; -} - -#endif /* JNI_FILE_H_ */ diff --git a/test-app/runtime/src/main/cpp/JsV8InspectorClient.cpp b/test-app/runtime/src/main/cpp/JsV8InspectorClient.cpp index d1ef7be83..6a64f079c 100644 --- a/test-app/runtime/src/main/cpp/JsV8InspectorClient.cpp +++ b/test-app/runtime/src/main/cpp/JsV8InspectorClient.cpp @@ -16,9 +16,9 @@ #include "NativeScriptAssert.h" #include "NativeScriptPlatform.h" +#include "AppFiles.h" #include "ArgConverter.h" #include "Constants.h" -#include "File.h" #include "Util.h" #include "Utils.h" #include "WorkerInspectorClient.h" @@ -195,13 +195,10 @@ bool GetInt32(const json& value, long long& out) { // key at the top level. bool ShouldRewriteSourceMapURLs() { static bool disabled = []() { - int length = 0; - auto* data = static_cast(File::ReadBinary(Constants::APP_ROOT_FOLDER_PATH + "package.json", length)); - if (data == nullptr) { + std::string content; + if (!AppFiles::ReadText(AppFiles::AppRootPrefix() + "package.json", content)) { return false; } - std::string content(reinterpret_cast(data), length); - delete[] data; auto parsed = json::parse(content, nullptr, false); if (parsed.is_discarded() || !parsed.is_object()) { @@ -837,7 +834,7 @@ std::string JsV8InspectorClient::HandleLoadNetworkResource(long long msgId, cons auto addVariants = [&addCandidate](const std::string& p) { addCandidate(p); if (p[0] != '/') { - addCandidate(Constants::APP_ROOT_FOLDER_PATH + p); + addCandidate(AppFiles::AppRootPrefix() + p); } // /data/data/ and /data/user/0/ alias the same files; // urls may use either spelling depending on who built them. @@ -858,11 +855,7 @@ std::string JsV8InspectorClient::HandleLoadNetworkResource(long long msgId, cons } for (const auto& candidate : candidates) { - int length = 0; - auto* data = static_cast(File::ReadBinary(candidate, length)); - if (data != nullptr) { - content.assign(reinterpret_cast(data), length); - delete[] data; + if (AppFiles::ReadText(candidate, content)) { loaded = true; break; } diff --git a/test-app/runtime/src/main/cpp/MetadataNode.cpp b/test-app/runtime/src/main/cpp/MetadataNode.cpp index adb1b3ebe..33193fb2b 100644 --- a/test-app/runtime/src/main/cpp/MetadataNode.cpp +++ b/test-app/runtime/src/main/cpp/MetadataNode.cpp @@ -1,4 +1,5 @@ #include "MetadataNode.h" +#include "AppFiles.h" #include "RuntimeState.h" #include "NativeScriptAssert.h" #include "Constants.h" @@ -906,7 +907,7 @@ vector MetadataNode::SetInstanceMethodsFromS Local funcData = External::New(isolate, callbackData, v8::kExternalPointerTypeTagDefault); Local funcTemplate = NewNonConstructor(isolate, MethodCallback, funcData); auto func = funcTemplate->GetFunction(context).ToLocalChecked(); - std::string origin = Constants::APP_ROOT_FOLDER_PATH + GetOrCreateInternal(treeNode)->m_name; + std::string origin = AppFiles::AppRootPrefix() + GetOrCreateInternal(treeNode)->m_name; Local wrapperFunc = Wrap(isolate, func, methodName, origin, false /* isCtorFunc */); Local ctorFunc = ctorFuncTemplate->GetFunction(context).ToLocalChecked(); @@ -1069,7 +1070,7 @@ void MetadataNode::SetStaticMembers(Isolate* isolate, Local& ctorFunct MethodCallbackData* callbackData = nullptr; auto cache = GetMetadataNodeCache(isolate); - auto origin = Constants::APP_ROOT_FOLDER_PATH + GetOrCreateInternal(treeNode)->m_name; + auto origin = AppFiles::AppRootPrefix() + GetOrCreateInternal(treeNode)->m_name; //get candidates from static methods metadata auto staticMethodCout = *reinterpret_cast(curPtr); @@ -1286,7 +1287,7 @@ Local MetadataNode::GetConstructorFunctionTemplate(Isolate* is auto context = isolate->GetCurrentContext(); auto ctorFunc = ctorFuncTemplate->GetFunction(context).ToLocalChecked(); - auto origin = Constants::APP_ROOT_FOLDER_PATH + node->m_name; + auto origin = AppFiles::AppRootPrefix() + node->m_name; auto wrappedCtorFunc = Wrap(isolate, ctorFunc, node->m_treeNode->name, origin, true /* isCtorFunc */); @@ -2123,7 +2124,7 @@ bool MetadataNode::GetExtendLocation(v8::Isolate* isolate, string& extendLocatio normalized.resize(queryOrFragment); } - const string& appRoot = Constants::APP_ROOT_FOLDER_PATH; + const string& appRoot = AppFiles::AppRootPrefix(); if (!appRoot.empty()) { stripPrefix(normalized, appRoot); } @@ -2214,11 +2215,15 @@ void MetadataNode::BuildMetadata(const string& filesPath) { string baseDir = filesPath; baseDir.append("/metadata"); - DIR* dir = opendir(baseDir.c_str()); + // Unreadable while credential-encrypted storage is locked (direct boot). + // Metadata served from the APK has no directory of its own, so the files + // directory stands in for it. + const string& guardDir = AppFiles::LoadsCodeFromApk() ? filesPath : baseDir; + DIR* dir = opendir(guardDir.c_str()); if(dir == nullptr){ stringstream ss; - ss << "metadata folder couldn't be opened! (Error: "; + ss << (AppFiles::LoadsCodeFromApk() ? "files" : "metadata") << " folder couldn't be opened! (Error: "; ss << errno; ss << ") "; @@ -2247,13 +2252,14 @@ void MetadataNode::BuildMetadata(const string& filesPath) { throw NativeScriptException(ss.str()); } } + closedir(dir); string nodesFile = baseDir + "/treeNodeStream.dat"; string namesFile = baseDir + "/treeStringsStream.dat"; string valuesFile = baseDir + "/treeValueStream.dat"; - FILE* f = fopen(nodesFile.c_str(), "rb"); - if (f == nullptr) { + auto nodesReader = AppFiles::Open(nodesFile); + if (nodesReader == nullptr) { stringstream ss; ss << "metadata file (treeNodeStream.dat) couldn't be opened! (Error: "; ss << errno; @@ -2261,11 +2267,9 @@ void MetadataNode::BuildMetadata(const string& filesPath) { throw NativeScriptException(ss.str()); } - fseek(f, 0, SEEK_END); - int lenNodes = ftell(f); + int lenNodes = static_cast(nodesReader->size()); if (lenNodes < 0 || (static_cast(lenNodes) % sizeof(MetadataTreeNodeRawData)) != 0) { - fclose(f); stringstream ss; ss << "Metadata file " << nodesFile << " is " << lenNodes << " bytes, which is not a whole number of " @@ -2274,41 +2278,36 @@ void MetadataNode::BuildMetadata(const string& filesPath) { throw NativeScriptException(ss.str()); } char* nodes = new char[lenNodes]; - rewind(f); - fread(nodes, 1, lenNodes, f); - fclose(f); + nodesReader->read(nodes, lenNodes); + nodesReader.reset(); const int _512KB = 524288; - f = fopen(namesFile.c_str(), "rb"); - if (f == nullptr) { + auto namesReader = AppFiles::Open(namesFile); + if (namesReader == nullptr) { stringstream ss; ss << "metadata file (treeStringsStream.dat) couldn't be opened! (Error: "; ss << errno; ss << ") "; throw NativeScriptException(ss.str()); } - fseek(f, 0, SEEK_END); - int lenNames = ftell(f); + int lenNames = static_cast(namesReader->size()); char* names = new char[lenNames + _512KB]; - rewind(f); - fread(names, 1, lenNames, f); - fclose(f); + namesReader->read(names, lenNames); + namesReader.reset(); - f = fopen(valuesFile.c_str(), "rb"); - if (f == nullptr) { + auto valuesReader = AppFiles::Open(valuesFile); + if (valuesReader == nullptr) { stringstream ss; ss << "metadata file (treeValueStream.dat) couldn't be opened! (Error: "; ss << errno; ss << ") "; throw NativeScriptException(ss.str()); } - fseek(f, 0, SEEK_END); - int lenValues = ftell(f); + int lenValues = static_cast(valuesReader->size()); char* values = new char[lenValues + _512KB]; - rewind(f); - fread(values, 1, lenValues, f); - fclose(f); + valuesReader->read(values, lenValues); + valuesReader.reset(); timeval time2; gettimeofday(&time2, nullptr); diff --git a/test-app/runtime/src/main/cpp/ModuleInternal.cpp b/test-app/runtime/src/main/cpp/ModuleInternal.cpp index 31fae2222..d88d3737d 100644 --- a/test-app/runtime/src/main/cpp/ModuleInternal.cpp +++ b/test-app/runtime/src/main/cpp/ModuleInternal.cpp @@ -5,9 +5,9 @@ * Author: gatanasov */ #include "ModuleInternal.h" +#include "AppFiles.h" #include "ModuleInternalCallbacks.h" #include "BuiltinLoader.h" -#include "File.h" #include "HttpLoader.h" #include "JniLocalRef.h" #include "ArgConverter.h" @@ -23,13 +23,12 @@ #include "CallbackHandlers.h" #include "ManualInstrumentation.h" #include "Runtime.h" +#include "ScriptCache.h" #include "TraceLog.h" #include #include #include -#include #include -#include #include #include #include @@ -169,8 +168,6 @@ void ModuleInternal::Init(Isolate* isolate, const string& baseDir) { RESOLVE_PATH_METHOD_ID = env.GetStaticMethodID(MODULE_CLASS, "resolvePath", "(Ljava/lang/String;Ljava/lang/String;)Ljava/lang/String;"); NS_CHECK(RESOLVE_PATH_METHOD_ID != nullptr); - GET_APPLICATION_FILES_PATH_METHOD_ID = env.GetStaticMethodID(MODULE_CLASS, "getApplicationFilesPath", "()Ljava/lang/String;"); - NS_CHECK(GET_APPLICATION_FILES_PATH_METHOD_ID != nullptr); } m_isolate = isolate; @@ -209,7 +206,7 @@ void ModuleInternal::Init(Isolate* isolate, const string& baseDir) { globalRequire = GetRequireFunction(isolate, baseDir, RequireEvaluationOptions( ModuleEvaluationPolicy::kSyncStrict)); } else { - globalRequire = GetRequireFunction(isolate, Constants::APP_ROOT_FOLDER_PATH, + globalRequire = GetRequireFunction(isolate, AppFiles::AppRootPrefix(), RequireEvaluationOptions( ModuleEvaluationPolicy::kSyncStrict)); } @@ -684,7 +681,7 @@ Local ModuleInternal::LoadImpl(Isolate* isolate, const string& moduleNam if (!moduleName.empty() && moduleName[0] == '~') { // Convert ~/path to ApplicationPath/path std::string tail = moduleName.size() >= 2 && moduleName[1] == '/' ? moduleName.substr(2) : moduleName.substr(1); - resolvedModuleName = Constants::APP_ROOT_FOLDER_PATH + "/" + tail; + resolvedModuleName = AppFiles::AppRootPrefix() + tail; // For .mjs files with tilde paths, use resolved path directly if (Util::EndsWith(resolvedModuleName, ".mjs")) { @@ -869,7 +866,12 @@ Local ModuleInternal::LoadModule(Isolate* isolate, const string& moduleP // attributed to whatever `.so` happens to load next. NapiModules::ClaimPendingModule(); - auto handle = dlopen(modulePath.c_str(), RTLD_LAZY); + // dlopen needs a real file; a library served from the APK has none. + std::string libraryPath; + if (!AppFiles::RealFilePath(modulePath, libraryPath)) { + libraryPath = modulePath; + } + auto handle = dlopen(libraryPath.c_str(), RTLD_LAZY); if (handle == nullptr) { auto error = dlerror(); string errMsg(error != nullptr ? error : "dlopen failed for " + modulePath); @@ -971,33 +973,37 @@ Local