From 2a389c7ab0096a8e2292a5bd8e4fd4a5e5525c36 Mon Sep 17 00:00:00 2001 From: Android PowerUser Date: Fri, 19 Jun 2026 16:59:59 +0200 Subject: [PATCH 01/11] Fix webview sync, back button, termux mode, media picker, and system message expansion --- .../com/google/ai/sample/MainActivity.kt | 105 ++++++++++- .../com/google/ai/sample/WebViewBridge.kt | 80 ++++---- index.html | 174 +++++++++++++----- 3 files changed, 270 insertions(+), 89 deletions(-) diff --git a/app/src/main/kotlin/com/google/ai/sample/MainActivity.kt b/app/src/main/kotlin/com/google/ai/sample/MainActivity.kt index 10b02dc9..6178841c 100644 --- a/app/src/main/kotlin/com/google/ai/sample/MainActivity.kt +++ b/app/src/main/kotlin/com/google/ai/sample/MainActivity.kt @@ -112,6 +112,13 @@ import okhttp3.Request import androidx.lifecycle.ViewModelProvider import androidx.lifecycle.ViewModelStoreOwner import com.google.ai.sample.GenerativeViewModelFactory +import androidx.activity.result.PickVisualMediaRequest +import android.graphics.drawable.BitmapDrawable +import android.media.MediaMetadataRetriever +import coil.ImageLoader +import coil.request.ImageRequest +import coil.request.SuccessResult +import coil.size.Precision class MainActivity : ComponentActivity() { @@ -143,6 +150,7 @@ class MainActivity : ComponentActivity() { private lateinit var mediaProjectionManager: MediaProjectionManager private lateinit var mediaProjectionLauncher: ActivityResultLauncher private lateinit var webRtcMediaProjectionLauncher: ActivityResultLauncher + private lateinit var pickMediaLauncher: ActivityResultLauncher private var currentScreenInfoForScreenshot: String? = null @@ -255,6 +263,15 @@ class MainActivity : ComponentActivity() { ) { result -> handleWebRtcMediaProjectionResult(result.resultCode, result.data) } + + pickMediaLauncher = registerForActivityResult(ActivityResultContracts.PickVisualMedia()) { uri -> + uri?.let { + Log.d(TAG, "Selected image URI from picker: $it") + webViewInstance?.post { + webViewInstance?.evaluateJavascript("window.onImagePicked('$it')", null) + } + } + } } private fun handleMediaProjectionResult(resultCode: Int, resultData: Intent?) { @@ -1289,15 +1306,74 @@ class MainActivity : ComponentActivity() { /** * Called by [WebViewBridge] when the user sends a chat message from the WebView UI. - * The WebView UI currently doesn't support attaching images, so this is always called - * with an empty image list. + * Supports passing a list of media URIs selected via the + button. */ - fun sendMessageFromWebView(text: String) { - Log.d(TAG, "sendMessageFromWebView called.") - photoReasoningViewModel?.reason( - userInput = text, - selectedImages = emptyList() - ) + fun sendMessageFromWebView(text: String, selectedImages: List) { + Log.d(TAG, "sendMessageFromWebView called with ${selectedImages.size} images.") + lifecycleScope.launch { + val bitmaps = selectedImages.mapNotNull { uri -> + uriToBitmap(uri) + } + photoReasoningViewModel?.reason( + userInput = text, + selectedImages = bitmaps, + screenInfoForPrompt = null, + imageUrisForChat = selectedImages.map { it.toString() } + ) + } + } + + private suspend fun uriToBitmap(uri: Uri): Bitmap? = withContext(Dispatchers.IO) { + val mimeType = contentResolver.getType(uri).orEmpty() + if (mimeType.startsWith("video/")) { + return@withContext extractVideoFrame(uri) + } + + val imageLoader = ImageLoader.Builder(this@MainActivity).build() + val imageRequest = ImageRequest.Builder(this@MainActivity) + .data(uri) + .precision(Precision.EXACT) + .build() + return@withContext try { + val result = imageLoader.execute(imageRequest) + if (result is SuccessResult) (result.drawable as? BitmapDrawable)?.bitmap else null + } catch (e: Exception) { + null + } + } + + private fun extractVideoFrame(uri: Uri): Bitmap? { + val retriever = MediaMetadataRetriever() + return try { + retriever.setDataSource(this, uri) + retriever.getFrameAtTime(0, MediaMetadataRetriever.OPTION_CLOSEST_SYNC) + } catch (e: Exception) { + Log.e(TAG, "Error extracting video frame for URI: $uri", e) + null + } finally { + retriever.release() + } + } + + fun openImagePicker() { + Log.d(TAG, "openImagePicker called via Bridge.") + pickMediaLauncher.launch(PickVisualMediaRequest(ActivityResultContracts.PickVisualMedia.ImageAndVideo)) + } + + override fun onBackPressed() { + val wv = webViewInstance + // Wenn wir nicht im WebView-Inhalt sind (htmlContent == null), nutzen wir standard back. + // Wenn WebView aktiv ist, fragen wir JS ob es ein "back" innerhalb der UI gibt. + if (wv != null && wv.visibility == View.VISIBLE) { + wv.evaluateJavascript("window.onBackPressed && window.onBackPressed()") { result -> + // JS gibt "true" zurück wenn es den Event konsumiert hat, sonst "false" oder "null" + if (result != "true") { + runOnUiThread { super.onBackPressed() } + } + } + } else { + super.onBackPressed() + } } /** @@ -1316,6 +1392,12 @@ class MainActivity : ComponentActivity() { fun setTermuxBackgroundFromWebView(background: Boolean) { Log.d(TAG, "setTermuxBackgroundFromWebView called with background=$background") TermuxExecutionModePreferences.setExecuteInBackground(this, background) + val toastMessage = if (background) { + "Termux commands are executed in the background" + } else { + "Termux commands are executed in the foreground" + } + Toast.makeText(this, toastMessage, Toast.LENGTH_SHORT).show() } /** @@ -1368,6 +1450,13 @@ class MainActivity : ComponentActivity() { } } } + lifecycleScope.launch { + vm.systemMessage.collect { msg -> + wv.post { + wv.evaluateJavascript("window.onSystemMessageChanged && window.onSystemMessageChanged('${escapeForJs(msg)}')", null) + } + } + } } private fun registerNetworkCallback() { diff --git a/app/src/main/kotlin/com/google/ai/sample/WebViewBridge.kt b/app/src/main/kotlin/com/google/ai/sample/WebViewBridge.kt index b07334a0..cca41507 100644 --- a/app/src/main/kotlin/com/google/ai/sample/WebViewBridge.kt +++ b/app/src/main/kotlin/com/google/ai/sample/WebViewBridge.kt @@ -6,6 +6,8 @@ import android.webkit.JavascriptInterface import android.webkit.WebView import com.google.ai.sample.feature.multimodal.PhotoReasoningUiState import com.google.ai.sample.util.GenerationSettingsPreferences +import com.google.ai.sample.util.SystemMessageEntry +import com.google.ai.sample.util.SystemMessageEntryPreferences import kotlinx.coroutines.CoroutineScope import kotlinx.coroutines.Dispatchers import kotlinx.coroutines.launch @@ -21,19 +23,21 @@ class WebViewBridge(private val mainActivity: MainActivity) { @JavascriptInterface fun getSystemMessage(): String { - val vm = mainActivity.getPhotoReasoningViewModel() - if (vm != null) return vm.systemMessage.value - return context.getSharedPreferences(PREFS_WEBVIEW, Context.MODE_PRIVATE) - .getString(KEY_SYS_MSG, "") ?: "" + return mainActivity.getPhotoReasoningViewModel()?.systemMessage?.value ?: "" } @JavascriptInterface fun setSystemMessage(message: String) { - context.getSharedPreferences(PREFS_WEBVIEW, Context.MODE_PRIVATE) - .edit().putString(KEY_SYS_MSG, message).apply() mainActivity.getPhotoReasoningViewModel()?.updateSystemMessage(message, context) } + @JavascriptInterface + fun restoreSystemMessage() { + mainActivity.runOnUiThread { + mainActivity.getPhotoReasoningViewModel()?.restoreSystemMessage(context) + } + } + // ── Model Selection ─────────────────────────────────────────────────────── @JavascriptInterface @@ -112,44 +116,33 @@ class WebViewBridge(private val mainActivity: MainActivity) { @JavascriptInterface fun getDatabaseEntries(): String { - val prefs = context.getSharedPreferences(PREFS_WEBVIEW_DB, Context.MODE_PRIVATE) - return prefs.getString(KEY_DB_ENTRIES, "[]") ?: "[]" + val entries = SystemMessageEntryPreferences.loadEntries(context) + val arr = JSONArray() + entries.forEach { + arr.put(JSONObject().put("title", it.title).put("guide", it.guide)) + } + return arr.toString() } @JavascriptInterface fun addDatabaseEntry(title: String, guide: String) { - val prefs = context.getSharedPreferences(PREFS_WEBVIEW_DB, Context.MODE_PRIVATE) - val arr = JSONArray(prefs.getString(KEY_DB_ENTRIES, "[]") ?: "[]") - arr.put(JSONObject().put("title", title).put("guide", guide)) - prefs.edit().putString(KEY_DB_ENTRIES, arr.toString()).apply() + SystemMessageEntryPreferences.addEntry(context, SystemMessageEntry(title, guide)) } @JavascriptInterface fun updateDatabaseEntry(oldTitle: String, newTitle: String, guide: String) { - val prefs = context.getSharedPreferences(PREFS_WEBVIEW_DB, Context.MODE_PRIVATE) - val arr = JSONArray(prefs.getString(KEY_DB_ENTRIES, "[]") ?: "[]") - val newArr = JSONArray() - for (i in 0 until arr.length()) { - val obj = arr.getJSONObject(i) - if (obj.getString("title") == oldTitle) { - newArr.put(JSONObject().put("title", newTitle).put("guide", guide)) - } else { - newArr.put(obj) - } + val oldEntry = SystemMessageEntryPreferences.loadEntries(context).find { it.title == oldTitle } + if (oldEntry != null) { + SystemMessageEntryPreferences.updateEntry(context, oldEntry, SystemMessageEntry(newTitle, guide)) } - prefs.edit().putString(KEY_DB_ENTRIES, newArr.toString()).apply() } @JavascriptInterface fun deleteDatabaseEntry(title: String) { - val prefs = context.getSharedPreferences(PREFS_WEBVIEW_DB, Context.MODE_PRIVATE) - val arr = JSONArray(prefs.getString(KEY_DB_ENTRIES, "[]") ?: "[]") - val newArr = JSONArray() - for (i in 0 until arr.length()) { - val obj = arr.getJSONObject(i) - if (obj.getString("title") != title) newArr.put(obj) + val entry = SystemMessageEntryPreferences.loadEntries(context).find { it.title == title } + if (entry != null) { + SystemMessageEntryPreferences.deleteEntry(context, entry) } - prefs.edit().putString(KEY_DB_ENTRIES, newArr.toString()).apply() } // ── Generation Settings ─────────────────────────────────────────────────── @@ -189,7 +182,22 @@ class WebViewBridge(private val mainActivity: MainActivity) { @JavascriptInterface fun sendMessage(text: String) { mainActivity.runOnUiThread { - mainActivity.sendMessageFromWebView(text) + mainActivity.sendMessageFromWebView(text, emptyList()) + } + } + + @JavascriptInterface + fun sendMessageWithImages(text: String, urisCsv: String) { + val uris = urisCsv.split(",").filter { it.isNotBlank() }.map { android.net.Uri.parse(it) } + mainActivity.runOnUiThread { + mainActivity.sendMessageFromWebView(text, uris) + } + } + + @JavascriptInterface + fun pickImage() { + mainActivity.runOnUiThread { + mainActivity.openImagePicker() } } @@ -264,14 +272,14 @@ class WebViewBridge(private val mainActivity: MainActivity) { } } + @JavascriptInterface + fun getTermuxBackground(): Boolean { + return com.google.ai.sample.util.TermuxExecutionModePreferences.executeInBackground(context) + } + // ── Helpers ─────────────────────────────────────────────────────────────── companion object { - private const val PREFS_WEBVIEW = "webview_prefs" - private const val PREFS_WEBVIEW_DB = "webview_db" - private const val KEY_SYS_MSG = "sysMsg" - private const val KEY_DB_ENTRIES = "entries" - fun jsEscape(s: String): String = s.replace("\\", "\\\\") .replace("'", "\\'") diff --git a/index.html b/index.html index 514fd860..4b24b7bb 100644 --- a/index.html +++ b/index.html @@ -116,11 +116,13 @@ .drop-item.drop-section:hover{background:transparent} /* ── System message card ─────────────────────────────────── */ -#sys-msg-textarea{width:100%;border:1px solid var(--input-border);border-radius:6px;padding:10px 12px;font-size:13px;font-family:inherit;background:var(--input-bg);color:var(--text-primary);outline:none;resize:none;line-height:1.5;overflow-y:auto;margin-top:10px;transition:max-height .2s ease} +#chat-sys-wrap .card-accent{transition:all .3s ease;overflow:hidden} +#sys-msg-textarea{width:100%;border:1px solid var(--input-border);border-radius:6px;padding:10px 12px;font-size:13px;font-family:inherit;background:var(--input-bg);color:var(--text-primary);outline:none;resize:none;line-height:1.5;overflow-y:auto;display:block;transition:height .3s ease} #sys-msg-textarea:focus{border-color:var(--primary)} -#sys-msg-textarea.collapsed{max-height:72px} -#sys-msg-textarea.expanded-keyboard{max-height:45vh} -#sys-msg-textarea.expanded-full{max-height:80vh} + +#sys-msg-textarea.collapsed{height:120px} +#sys-msg-textarea.expanded-keyboard{height:35vh} +#sys-msg-textarea.expanded-full{height:78vh} /* ── Chat bubbles ────────────────────────────────────────── */ .bubble-wrap{display:flex;align-items:flex-start;gap:6px;margin:6px 0} @@ -438,6 +440,11 @@ /* System message */ getSystemMessage:()=> inAndroid ? Android.getSystemMessage() : (LS.getItem('sysMsg')||DEFAULT_SYSTEM_MSG), setSystemMessage:(m)=>{ inAndroid ? Android.setSystemMessage(m) : LS.setItem('sysMsg',m); }, + restoreSystemMessage:()=>{ if(inAndroid) Android.restoreSystemMessage(); else LS.setItem('sysMsg', DEFAULT_SYSTEM_MSG); }, + + /* Termux background */ + getTermuxBackground:()=> inAndroid ? Android.getTermuxBackground() : (LS.getItem('termuxBg')==='true'), + setTermuxBackground:(b)=>{ inAndroid ? Android.setTermuxBackground(b) : LS.setItem('termuxBg',String(b)); }, /* Model selection */ getSelectedModelId:()=> inAndroid ? Android.getSelectedModelId() : (LS.getItem('modelId')||'PUTER_QWEN2_5_VL_72B'), @@ -485,6 +492,8 @@ /* Chat */ sendMessage:(text)=>{ if(inAndroid) Android.sendMessage(text); else addModelBubble('[Demo] Processing: '+text,false); }, + sendMessageWithImages:(text,urisCsv)=>{ if(inAndroid) Android.sendMessageWithImages(text,urisCsv); else addModelBubble('[Demo] Processing with images: '+urisCsv,false); }, + pickImage:()=>{ if(inAndroid) Android.pickImage(); else document.getElementById('image-file-input').click(); }, clearChatHistory:()=>{ if(inAndroid) Android.clearChatHistory(); }, stopGeneration:()=>{ if(inAndroid) Android.stopGeneration(); }, isGenerationRunning:()=> inAndroid ? Android.isGenerationRunning() : false, @@ -590,21 +599,48 @@ // System message document.getElementById('sys-msg-textarea').value = Bridge.getSystemMessage(); + // Termux background setting + termuxBackground = Bridge.getTermuxBackground(); + document.getElementById('tb-btn').textContent = termuxBackground ? 'TB' : 'TF'; + // Keyboard height: shift chat-bottom above keyboard using visualViewport if (window.visualViewport) { - window.visualViewport.addEventListener('resize', onViewportResize); - window.visualViewport.addEventListener('scroll', onViewportResize); + window.visualViewport.addEventListener('resize', () => { + onViewportResize(); + updateSysTextareaClass(); + }); + window.visualViewport.addEventListener('scroll', () => { + onViewportResize(); + updateSysTextareaClass(); + }); } // Back-button: listen for popstate to go back to menu window.addEventListener('popstate', (e) => { const chatActive = document.getElementById('screen-chat').classList.contains('active'); if (chatActive) { - _showMenu(); + navigateToMenu(); } }); }); +window.onBackPressed = function() { + if (document.getElementById('edit-entry-popup').classList.contains('open')) { + closeEditEntry(); + return "true"; + } + if (document.getElementById('db-popup').classList.contains('open')) { + closeDatabase(); + return "true"; + } + const chatActive = document.getElementById('screen-chat').classList.contains('active'); + if (chatActive) { + navigateToMenu(); + return "true"; + } + return "false"; +}; + function onViewportResize() { const vv = window.visualViewport; const chatBottom = document.getElementById('chat-bottom'); @@ -676,7 +712,8 @@ document.getElementById('screen-menu').classList.remove('active'); document.getElementById('screen-chat').classList.add('active'); // Sync system message fresh from app - document.getElementById('sys-msg-textarea').value = Bridge.getSystemMessage(); + const msg = Bridge.getSystemMessage(); + document.getElementById('sys-msg-textarea').value = msg; scrollToBottom(); } function navigateToMenu() { @@ -827,9 +864,10 @@ } /* ════════════════════════════════════════════════════════ - SYSTEM MESSAGE + SYSTEM MESSAGE & KEYBOARD / VIEWPORT SYNC ════════════════════════════════════════════════════════ */ let sysTimeout = null; + function onSystemMessageChange() { clearTimeout(sysTimeout); autoGrowTextarea(document.getElementById('sys-msg-textarea')); @@ -837,26 +875,60 @@ Bridge.setSystemMessage(document.getElementById('sys-msg-textarea').value); }, 600); } -function onSysTextareaFocus() { + +function updateSysTextareaClass() { const ta = document.getElementById('sys-msg-textarea'); - const vv = window.visualViewport; - const keyH = vv ? (window.innerHeight - vv.height - vv.offsetTop) : 0; - ta.className = keyH > 10 ? 'expanded-keyboard' : 'expanded-full'; - autoGrowTextarea(ta); + if (document.activeElement !== ta) return; + const isKeyboard = window.visualViewport && (window.visualViewport.height < window.innerHeight * 0.88); + ta.className = isKeyboard ? 'expanded-keyboard' : 'expanded-full'; +} + +function onSysTextareaFocus() { + updateSysTextareaClass(); + autoGrowTextarea(document.getElementById('sys-msg-textarea')); } + function onSysTextareaBlur() { const ta = document.getElementById('sys-msg-textarea'); - ta.className = 'collapsed'; + setTimeout(() => { + if (document.activeElement !== ta) { + ta.className = 'collapsed'; + } + }, 150); } + function restoreSystemMessage() { - const ta = document.getElementById('sys-msg-textarea'); - ta.value = DEFAULT_SYSTEM_MSG; - Bridge.setSystemMessage(DEFAULT_SYSTEM_MSG); - autoGrowTextarea(ta); + Bridge.restoreSystemMessage(); } + // Android can call this when system message changes externally window.onSystemMessageChanged = function(msg) { - document.getElementById('sys-msg-textarea').value = msg; + const ta = document.getElementById('sys-msg-textarea'); + if (document.activeElement !== ta) { + ta.value = msg; + autoGrowTextarea(ta); + } +}; + +// Android System Back Button Handler (returns Boolean to WebViewBridge) +window.onBackPressed = function() { + if (document.getElementById('popup-edit-entry').classList.contains('open')) { + closePopup('popup-edit-entry'); + return true; + } + if (document.getElementById('popup-db').classList.contains('open')) { + closePopup('popup-db'); + return true; + } + if (document.getElementById('popup-apikey').classList.contains('open')) { + closePopup('popup-apikey'); + return true; + } + if (document.getElementById('screen-chat').classList.contains('active')) { + navigateToMenu(); + return true; + } + return false; }; /* ════════════════════════════════════════════════════════ @@ -912,17 +984,20 @@ const val = ta.value.trim(); const icon = document.getElementById('send-icon'); icon.className = val ? 'send-icon active' : 'send-icon'; + + if (document.activeElement === ta) { + ta.placeholder = val ? 'Task' : 'Describe step by step if complicated tasks have to be solved'; + } else { + ta.placeholder = 'Task'; + } + autoGrowTextarea(ta); } function onTaskInputFocus() { - const ta = document.getElementById('task-input'); - if (!ta.value.trim()) { - ta.placeholder = 'Describe step by step if complicated tasks have to be solved'; - } + onTaskInput(); } function onTaskInputBlur() { - const ta = document.getElementById('task-input'); - ta.placeholder = 'Task'; + onTaskInput(); } function onTaskKey(e) { if (e.key === 'Enter' && !e.shiftKey) { e.preventDefault(); sendMessage(); } @@ -930,15 +1005,15 @@ function sendMessage() { const input = document.getElementById('task-input'); const text = input.value.trim(); - if (!text) return; - addUserBubble(text); + if (!text && selectedImages.length === 0) return; + addUserBubble(text || "(attached images)"); input.value = ''; input.style.height = ''; onTaskInput(); // Pass image URIs to Android if any - if (inAndroid && Android.sendMessageWithImages && selectedImages.length > 0) { - const uris = selectedImages.map(img => img.uri || img.objectUrl).join(','); - Android.sendMessageWithImages(text, uris); + if (selectedImages.length > 0) { + const urisCsv = selectedImages.map(img => img.uri || img.objectUrl).join(','); + Bridge.sendMessageWithImages(text, urisCsv); } else { Bridge.sendMessage(text); } @@ -954,11 +1029,7 @@ let selectedImages = []; function pickImage() { - if (inAndroid && Android.pickImage) { - Android.pickImage(); - } else { - document.getElementById('image-file-input').click(); - } + Bridge.pickImage(); } function onFilesPicked(event) { @@ -1009,11 +1080,7 @@ function toggleTermuxMode() { termuxBackground = !termuxBackground; document.getElementById('tb-btn').textContent = termuxBackground ? 'TB' : 'TF'; - if (inAndroid && Android.setTermuxBackground) Android.setTermuxBackground(termuxBackground); - const msg = termuxBackground - ? 'Termux commands are executed in the background' - : 'Termux commands are executed in the foreground'; - showToast(msg); + Bridge.setTermuxBackground(termuxBackground); } /* ════════════════════════════════════════════════════════ @@ -1094,6 +1161,7 @@ function renderDbList() { const list = document.getElementById('db-list'); const entries = Bridge.getDatabaseEntries(); + list.innerHTML = ''; if (!entries.length) { list.innerHTML='
No entries yet. Tap "+ New Entry" to add one.
'; @@ -1102,12 +1170,27 @@ entries.forEach(e => { const row = document.createElement('div'); row.className='db-entry'; - row.innerHTML=`${escHtml(e.title)} - `; - row.addEventListener('click', () => openEditEntry(e)); + row.innerHTML=` +
+
${escHtml(e.title)}
+
${escHtml(e.guide.substring(0,60))}...
+
+
+ + +
`; + row.querySelector('div').onclick = () => openEditEntry(e); list.appendChild(row); }); } + +function copyToSystemMessage(text) { + const ta = document.getElementById('sys-msg-textarea'); + ta.value = text; + Bridge.setSystemMessage(text); + autoGrowTextarea(ta); + closePopup('popup-db'); +} function deleteEntry(title) { if (!confirm('Delete "'+title+'"?')) return; Bridge.deleteDatabaseEntry(title); @@ -1127,7 +1210,8 @@ if (editingEntryOriginalTitle) { Bridge.updateDatabaseEntry(editingEntryOriginalTitle, title, guide); } else { - const exists = Bridge.getDatabaseEntries().some(e=>e.title.toLowerCase()===title.toLowerCase()); + const entries = Bridge.getDatabaseEntries(); + const exists = entries.some(e=>e.title.toLowerCase()===title.toLowerCase()); if (exists) { alert('An entry with this title already exists.'); return; } Bridge.addDatabaseEntry(title, guide); } From c3d08680e5ad0d2c5cfe636631481669d86fe2e0 Mon Sep 17 00:00:00 2001 From: Android PowerUser Date: Sat, 20 Jun 2026 09:55:44 +0200 Subject: [PATCH 02/11] Fix WebView, Bridge, keyboard overlay, media picker, and sync issues --- app/src/main/AndroidManifest.xml | 3 +- .../com/google/ai/sample/MainActivity.kt | 15 ++- index.html | 114 ++++++++++-------- 3 files changed, 73 insertions(+), 59 deletions(-) diff --git a/app/src/main/AndroidManifest.xml b/app/src/main/AndroidManifest.xml index 137225d8..7788ed6d 100644 --- a/app/src/main/AndroidManifest.xml +++ b/app/src/main/AndroidManifest.xml @@ -43,7 +43,8 @@ android:exported="true" android:label="@string/app_name" android:theme="@style/Theme.Emptything" - android:launchMode="singleTop"> + android:launchMode="singleTop" + android:windowSoftInputMode="adjustResize"> diff --git a/app/src/main/kotlin/com/google/ai/sample/MainActivity.kt b/app/src/main/kotlin/com/google/ai/sample/MainActivity.kt index 6178841c..018bcfec 100644 --- a/app/src/main/kotlin/com/google/ai/sample/MainActivity.kt +++ b/app/src/main/kotlin/com/google/ai/sample/MainActivity.kt @@ -266,9 +266,10 @@ class MainActivity : ComponentActivity() { pickMediaLauncher = registerForActivityResult(ActivityResultContracts.PickVisualMedia()) { uri -> uri?.let { - Log.d(TAG, "Selected image URI from picker: $it") + Log.d(TAG, "Selected image/video URI from picker: $it") + val isVideo = contentResolver.getType(it)?.startsWith("video/") == true webViewInstance?.post { - webViewInstance?.evaluateJavascript("window.onImagePicked('$it')", null) + webViewInstance?.evaluateJavascript("window.onImagePicked('$it', $isVideo)", null) } } } @@ -720,8 +721,8 @@ class MainActivity : ComponentActivity() { settings.javaScriptEnabled = true settings.domStorageEnabled = true settings.databaseEnabled = false - settings.allowFileAccess = false - settings.allowContentAccess = false + settings.allowFileAccess = true + settings.allowContentAccess = true settings.mixedContentMode = WebSettings.MIXED_CONTENT_NEVER_ALLOW settings.setSupportZoom(true) settings.builtInZoomControls = true @@ -739,6 +740,9 @@ class MainActivity : ComponentActivity() { override fun onPageFinished(view: WebView?, url: String?) { super.onPageFinished(view, url) Log.d(TAG, "WebView page rendered: {}".format(url)) + view?.post { + view.evaluateJavascript("window.onAndroidReady && window.onAndroidReady()", null) + } observeViewModelForWebView() } @@ -1367,7 +1371,8 @@ class MainActivity : ComponentActivity() { if (wv != null && wv.visibility == View.VISIBLE) { wv.evaluateJavascript("window.onBackPressed && window.onBackPressed()") { result -> // JS gibt "true" zurück wenn es den Event konsumiert hat, sonst "false" oder "null" - if (result != "true") { + val cleanedResult = result?.replace("\"", "")?.trim() + if (cleanedResult != "true") { runOnUiThread { super.onBackPressed() } } } diff --git a/index.html b/index.html index 4b24b7bb..4296ca55 100644 --- a/index.html +++ b/index.html @@ -434,78 +434,78 @@ Falls back to localStorage when running in a browser. ════════════════════════════════════════════════════════ */ const LS = window.localStorage; -const inAndroid = typeof window.Android !== 'undefined'; +const getInAndroid = () => typeof window.Android !== 'undefined'; const Bridge = { /* System message */ - getSystemMessage:()=> inAndroid ? Android.getSystemMessage() : (LS.getItem('sysMsg')||DEFAULT_SYSTEM_MSG), - setSystemMessage:(m)=>{ inAndroid ? Android.setSystemMessage(m) : LS.setItem('sysMsg',m); }, - restoreSystemMessage:()=>{ if(inAndroid) Android.restoreSystemMessage(); else LS.setItem('sysMsg', DEFAULT_SYSTEM_MSG); }, + getSystemMessage:()=> getInAndroid() ? Android.getSystemMessage() : (LS.getItem('sysMsg')||DEFAULT_SYSTEM_MSG), + setSystemMessage:(m)=>{ getInAndroid() ? Android.setSystemMessage(m) : LS.setItem('sysMsg',m); }, + restoreSystemMessage:()=>{ if(getInAndroid()) Android.restoreSystemMessage(); else LS.setItem('sysMsg', DEFAULT_SYSTEM_MSG); }, /* Termux background */ - getTermuxBackground:()=> inAndroid ? Android.getTermuxBackground() : (LS.getItem('termuxBg')==='true'), - setTermuxBackground:(b)=>{ inAndroid ? Android.setTermuxBackground(b) : LS.setItem('termuxBg',String(b)); }, + getTermuxBackground:()=> getInAndroid() ? Android.getTermuxBackground() : (LS.getItem('termuxBg')==='true'), + setTermuxBackground:(b)=>{ getInAndroid() ? Android.setTermuxBackground(b) : LS.setItem('termuxBg',String(b)); }, /* Model selection */ - getSelectedModelId:()=> inAndroid ? Android.getSelectedModelId() : (LS.getItem('modelId')||'PUTER_QWEN2_5_VL_72B'), - setSelectedModel:(id)=>{ inAndroid ? Android.setSelectedModel(id) : LS.setItem('modelId',id); }, + getSelectedModelId:()=> getInAndroid() ? Android.getSelectedModelId() : (LS.getItem('modelId')||'PUTER_QWEN2_5_VL_72B'), + setSelectedModel:(id)=>{ getInAndroid() ? Android.setSelectedModel(id) : LS.setItem('modelId',id); }, /* API Keys */ - getAllApiKeys:(prov)=> JSON.parse(inAndroid ? Android.getAllApiKeys(prov) : (LS.getItem('keys_'+prov)||'[]')), + getAllApiKeys:(prov)=> JSON.parse(getInAndroid() ? Android.getAllApiKeys(prov) : (LS.getItem('keys_'+prov)||'[]')), addApiKey:(key,prov)=>{ - if(inAndroid) return Android.addApiKey(key,prov); + if(getInAndroid()) return Android.addApiKey(key,prov); const k=Bridge.getAllApiKeys(prov); if(k.includes(key)) return false; k.push(key); LS.setItem('keys_'+prov,JSON.stringify(k)); return true; }, removeApiKey:(key,prov)=>{ - if(inAndroid){Android.removeApiKey(key,prov);return;} + if(getInAndroid()){Android.removeApiKey(key,prov);return;} const k=Bridge.getAllApiKeys(prov).filter(x=>x!==key); LS.setItem('keys_'+prov,JSON.stringify(k)); }, - getCurrentKeyIndex:(prov)=> inAndroid ? Android.getCurrentKeyIndex(prov) : parseInt(LS.getItem('kIdx_'+prov)||'0'), - setCurrentKeyIndex:(idx,prov)=>{ inAndroid ? Android.setCurrentKeyIndex(idx,prov) : LS.setItem('kIdx_'+prov,String(idx)); }, + getCurrentKeyIndex:(prov)=> getInAndroid() ? Android.getCurrentKeyIndex(prov) : parseInt(LS.getItem('kIdx_'+prov)||'0'), + setCurrentKeyIndex:(idx,prov)=>{ getInAndroid() ? Android.setCurrentKeyIndex(idx,prov) : LS.setItem('kIdx_'+prov,String(idx)); }, /* Database entries */ - getDatabaseEntries:()=> JSON.parse(inAndroid ? Android.getDatabaseEntries() : (LS.getItem('dbEntries')||'[]')), + getDatabaseEntries:()=> JSON.parse(getInAndroid() ? Android.getDatabaseEntries() : (LS.getItem('dbEntries')||'[]')), addDatabaseEntry:(title,guide)=>{ - if(inAndroid){Android.addDatabaseEntry(title,guide);return;} + if(getInAndroid()){Android.addDatabaseEntry(title,guide);return;} const e=Bridge.getDatabaseEntries(); e.push({title,guide}); LS.setItem('dbEntries',JSON.stringify(e)); }, updateDatabaseEntry:(oldTitle,title,guide)=>{ - if(inAndroid){Android.updateDatabaseEntry(oldTitle,title,guide);return;} + if(getInAndroid()){Android.updateDatabaseEntry(oldTitle,title,guide);return;} const e=Bridge.getDatabaseEntries().map(x=>x.title===oldTitle?{title,guide}:x); LS.setItem('dbEntries',JSON.stringify(e)); }, deleteDatabaseEntry:(title)=>{ - if(inAndroid){Android.deleteDatabaseEntry(title);return;} + if(getInAndroid()){Android.deleteDatabaseEntry(title);return;} const e=Bridge.getDatabaseEntries().filter(x=>x.title!==title); LS.setItem('dbEntries',JSON.stringify(e)); }, /* Generation settings */ getGenerationSettings:(id)=>{ const def={temperature:0,topP:0,topK:1}; - if(inAndroid) return JSON.parse(Android.getGenerationSettings(id)||'{}'); + if(getInAndroid()) return JSON.parse(Android.getGenerationSettings(id)||'{}'); return JSON.parse(LS.getItem('gen_'+id)||JSON.stringify(def)); }, saveGenerationSettings:(id,temp,topP,topK)=>{ - if(inAndroid){Android.saveGenerationSettings(id,temp,topP,topK);return;} + if(getInAndroid()){Android.saveGenerationSettings(id,temp,topP,topK);return;} LS.setItem('gen_'+id,JSON.stringify({temperature:temp,topP,topK})); }, /* Chat */ - sendMessage:(text)=>{ if(inAndroid) Android.sendMessage(text); else addModelBubble('[Demo] Processing: '+text,false); }, - sendMessageWithImages:(text,urisCsv)=>{ if(inAndroid) Android.sendMessageWithImages(text,urisCsv); else addModelBubble('[Demo] Processing with images: '+urisCsv,false); }, - pickImage:()=>{ if(inAndroid) Android.pickImage(); else document.getElementById('image-file-input').click(); }, - clearChatHistory:()=>{ if(inAndroid) Android.clearChatHistory(); }, - stopGeneration:()=>{ if(inAndroid) Android.stopGeneration(); }, - isGenerationRunning:()=> inAndroid ? Android.isGenerationRunning() : false, - isOfflineModelLoaded:()=> inAndroid ? Android.isOfflineModelLoaded() : false, + sendMessage:(text)=>{ if(getInAndroid()) Android.sendMessage(text); else addModelBubble('[Demo] Processing: '+text,false); }, + sendMessageWithImages:(text,urisCsv)=>{ if(getInAndroid()) Android.sendMessageWithImages(text,urisCsv); else addModelBubble('[Demo] Processing with images: '+urisCsv,false); }, + pickImage:()=>{ if(getInAndroid()) Android.pickImage(); else document.getElementById('image-file-input').click(); }, + clearChatHistory:()=>{ if(getInAndroid()) Android.clearChatHistory(); }, + stopGeneration:()=>{ if(getInAndroid()) Android.stopGeneration(); }, + isGenerationRunning:()=> getInAndroid() ? Android.isGenerationRunning() : false, + isOfflineModelLoaded:()=> getInAndroid() ? Android.isOfflineModelLoaded() : false, /* Backend */ - getBackendPreference:()=> inAndroid ? Android.getBackendPreference() : (LS.getItem('backend')||'GPU'), - setBackendPreference:(b)=>{ inAndroid ? Android.setBackendPreference(b) : LS.setItem('backend',b); }, + getBackendPreference:()=> getInAndroid() ? Android.getBackendPreference() : (LS.getItem('backend')||'GPU'), + setBackendPreference:(b)=>{ getInAndroid() ? Android.setBackendPreference(b) : LS.setItem('backend',b); }, /* Donation / trial */ - initiateDonation:()=>{ if(inAndroid) Android.initiateDonation(); else alert('Subscription flow would open here'); }, - isPurchased:()=> inAndroid ? Android.isPurchased() : false, + initiateDonation:()=>{ if(getInAndroid()) Android.initiateDonation(); else alert('Subscription flow would open here'); }, + isPurchased:()=> getInAndroid() ? Android.isPurchased() : false, }; /* ════════════════════════════════════════════════════════ @@ -624,22 +624,7 @@ }); }); -window.onBackPressed = function() { - if (document.getElementById('edit-entry-popup').classList.contains('open')) { - closeEditEntry(); - return "true"; - } - if (document.getElementById('db-popup').classList.contains('open')) { - closeDatabase(); - return "true"; - } - const chatActive = document.getElementById('screen-chat').classList.contains('active'); - if (chatActive) { - navigateToMenu(); - return "true"; - } - return "false"; -}; + function onViewportResize() { const vv = window.visualViewport; @@ -910,6 +895,22 @@ } }; +window.onAndroidReady = function() { + currentModelId = Bridge.getSelectedModelId(); + buildModelDropdown(); + applyModelSelection(currentModelId); + loadGenerationSettings(); + loadBackendPreference(); + updateDonationCard(); + + // System message + document.getElementById('sys-msg-textarea').value = Bridge.getSystemMessage(); + + // Termux background setting + termuxBackground = Bridge.getTermuxBackground(); + document.getElementById('tb-btn').textContent = termuxBackground ? 'TB' : 'TF'; +}; + // Android System Back Button Handler (returns Boolean to WebViewBridge) window.onBackPressed = function() { if (document.getElementById('popup-edit-entry').classList.contains('open')) { @@ -1036,15 +1037,16 @@ const files = Array.from(event.target.files); files.forEach(file => { const url = URL.createObjectURL(file); - selectedImages.push({ objectUrl: url, name: file.name }); + const isVideo = file.type.startsWith('video/'); + selectedImages.push({ objectUrl: url, name: file.name, isVideo: isVideo }); }); event.target.value = ''; renderImagePreviews(); } -// Called by Android after user picks a media item: window.onImagePicked(uri) -window.onImagePicked = function(uri) { - selectedImages.push({ uri: uri, name: uri.split('/').pop() }); +// Called by Android after user picks a media item: window.onImagePicked(uri, isVideo) +window.onImagePicked = function(uri, isVideo) { + selectedImages.push({ uri: uri, name: uri.split('/').pop(), isVideo: !!isVideo }); renderImagePreviews(); }; @@ -1054,9 +1056,15 @@ selectedImages.forEach((img, i) => { const wrap = document.createElement('div'); wrap.className = 'img-thumb-wrap'; - const src = img.objectUrl || img.uri; - wrap.innerHTML = `${escAttr(img.name)} - `; + if (img.isVideo) { + wrap.innerHTML = ` +
🎬
+ `; + } else { + const src = img.objectUrl || img.uri; + wrap.innerHTML = `${escAttr(img.name)} + `; + } row.appendChild(wrap); }); } From 0eb7544fb4b6683cb0bdc689964d1ded38d57c10 Mon Sep 17 00:00:00 2001 From: Android PowerUser Date: Sat, 20 Jun 2026 09:56:36 +0200 Subject: [PATCH 03/11] Point WebView content URLs to feature/webview-test branch --- app/src/main/kotlin/com/google/ai/sample/MainActivity.kt | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/app/src/main/kotlin/com/google/ai/sample/MainActivity.kt b/app/src/main/kotlin/com/google/ai/sample/MainActivity.kt index 018bcfec..28eb125c 100644 --- a/app/src/main/kotlin/com/google/ai/sample/MainActivity.kt +++ b/app/src/main/kotlin/com/google/ai/sample/MainActivity.kt @@ -612,7 +612,7 @@ class MainActivity : ComponentActivity() { private fun loadWebViewContent() { if (webViewHtmlContent != null) return - val htmlUrl = "https://raw.githubusercontent.com/Android-PowerUser/ScreenOperator/refs/heads/main/index.html" + val htmlUrl = "https://raw.githubusercontent.com/Android-PowerUser/ScreenOperator/refs/heads/feature/webview-test/index.html" lifecycleScope.launch(Dispatchers.IO) { if (webViewHtmlContent != null) return@launch try { @@ -759,7 +759,7 @@ class MainActivity : ComponentActivity() { this@MainActivity.webViewInstance = this addJavascriptInterface(WebViewBridge(this@MainActivity), "Android") loadDataWithBaseURL( - "https://raw.githubusercontent.com/Android-PowerUser/ScreenOperator/refs/heads/main/", + "https://raw.githubusercontent.com/Android-PowerUser/ScreenOperator/refs/heads/feature/webview-test/", htmlContent, "text/html", "UTF-8", From 12e0dd195ab5f1a63086f811725f61c4eb339c5e Mon Sep 17 00:00:00 2001 From: Android PowerUser <88908510+Android-PowerUser@users.noreply.github.com> Date: Thu, 25 Jun 2026 09:53:42 +0200 Subject: [PATCH 04/11] Fix: Load system message from preferences when ViewModel not initialized - Modified getSystemMessage() in WebViewBridge to automatically load system message from SystemMessagePreferences when ViewModel is not initialized and current message is empty - Added import for SystemMessagePreferences - This ensures system message is displayed on WebView startup without requiring manual restore - Does not call restoreSystemMessage() as per requirement, but loads directly from app data --- .../kotlin/com/google/ai/sample/WebViewBridge.kt | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/app/src/main/kotlin/com/google/ai/sample/WebViewBridge.kt b/app/src/main/kotlin/com/google/ai/sample/WebViewBridge.kt index cca41507..d7b672f2 100644 --- a/app/src/main/kotlin/com/google/ai/sample/WebViewBridge.kt +++ b/app/src/main/kotlin/com/google/ai/sample/WebViewBridge.kt @@ -8,6 +8,7 @@ import com.google.ai.sample.feature.multimodal.PhotoReasoningUiState import com.google.ai.sample.util.GenerationSettingsPreferences import com.google.ai.sample.util.SystemMessageEntry import com.google.ai.sample.util.SystemMessageEntryPreferences +import com.google.ai.sample.util.SystemMessagePreferences import kotlinx.coroutines.CoroutineScope import kotlinx.coroutines.Dispatchers import kotlinx.coroutines.launch @@ -23,7 +24,17 @@ class WebViewBridge(private val mainActivity: MainActivity) { @JavascriptInterface fun getSystemMessage(): String { - return mainActivity.getPhotoReasoningViewModel()?.systemMessage?.value ?: "" + val viewModel = mainActivity.getPhotoReasoningViewModel() + val currentMessage = viewModel?.systemMessage?.value ?: "" + + // If system message is empty and ViewModel is not initialized, load from preferences + if (currentMessage.isEmpty() && (viewModel?.isInitialized?.value == false)) { + val savedMessage = SystemMessagePreferences.loadSystemMessage(context) + Log.d(TAG, "getSystemMessage: Loading from preferences because ViewModel not initialized. Length: ${savedMessage.length}") + return savedMessage + } + + return currentMessage } @JavascriptInterface From fa927579c43af231fdfa159e916722df69f0e469 Mon Sep 17 00:00:00 2001 From: Claude Date: Thu, 25 Jun 2026 09:03:04 +0000 Subject: [PATCH 05/11] Make command syntax remotely overridable (no app release needed) Adds a mechanism so a new model's slightly different command syntax (e.g. "Click('...')" instead of "click(\"...\")") can be supported via a repo commit to command-patterns.json instead of patching CommandParser.kt and shipping a new app version. - CommandParser.CommandType is now public; CommandPatternConfig parses a remote JSON array of {id, commandType, regex} overrides. - An override can only attach a new regex to an EXISTING CommandType - the actual Command construction/execution logic is always the same compiled-in builder. No new action kind or custom code can be introduced this way. - WebViewBridge exposes setCommandPatternOverrides()/getCommandPatternOverrides(); the WebView fetches the optional command-patterns.json on window.onAndroidReady() and pushes it to the bridge. - CommandPatternOverridesPreferences persists the last received override JSON so it survives app restarts; PhotoReasoningApplication.onCreate() restores it. - Added unit tests covering: alternate syntax recognition, unknown commandType rejection, invalid regex rejection, and clearing overrides. Existing CommandParserTest cases are unaffected (overrides default to empty). Execution/queueing/guard logic in PhotoReasoningViewModel/AccessibilityCommandQueue is intentionally untouched - only the *pattern recognition* layer is now data-driven and remotely updatable. --- .../ai/sample/PhotoReasoningApplication.kt | 8 ++ .../com/google/ai/sample/WebViewBridge.kt | 23 ++++ .../google/ai/sample/util/CommandParser.kt | 115 +++++++++++++----- .../ai/sample/util/CommandPatternConfig.kt | 85 +++++++++++++ .../CommandPatternOverridesPreferences.kt | 38 ++++++ .../ai/sample/util/CommandParserTest.kt | 47 +++++++ command-patterns.json | 1 + docs/command-pattern-overrides.md | 64 ++++++++++ index.html | 19 +++ 9 files changed, 368 insertions(+), 32 deletions(-) create mode 100644 app/src/main/kotlin/com/google/ai/sample/util/CommandPatternConfig.kt create mode 100644 app/src/main/kotlin/com/google/ai/sample/util/CommandPatternOverridesPreferences.kt create mode 100644 command-patterns.json create mode 100644 docs/command-pattern-overrides.md diff --git a/app/src/main/kotlin/com/google/ai/sample/PhotoReasoningApplication.kt b/app/src/main/kotlin/com/google/ai/sample/PhotoReasoningApplication.kt index 444e2069..d7fee77e 100644 --- a/app/src/main/kotlin/com/google/ai/sample/PhotoReasoningApplication.kt +++ b/app/src/main/kotlin/com/google/ai/sample/PhotoReasoningApplication.kt @@ -37,5 +37,13 @@ class PhotoReasoningApplication : Application() { super.onCreate() instance = this Log.d(TAG, "Application created") + + // Re-apply any command pattern overrides that were previously received from the + // WebView bundle, so alternate command syntax for new models keeps working even + // before the WebView has re-fetched/re-applied its config in this session. + com.google.ai.sample.util.CommandPatternOverridesPreferences.load(this)?.let { savedJson -> + val applied = com.google.ai.sample.util.CommandParser.setRemotePatternOverrides(savedJson) + Log.d(TAG, "Restored $applied command pattern override(s) from preferences") + } } } diff --git a/app/src/main/kotlin/com/google/ai/sample/WebViewBridge.kt b/app/src/main/kotlin/com/google/ai/sample/WebViewBridge.kt index d7b672f2..696fe1d6 100644 --- a/app/src/main/kotlin/com/google/ai/sample/WebViewBridge.kt +++ b/app/src/main/kotlin/com/google/ai/sample/WebViewBridge.kt @@ -288,6 +288,29 @@ class WebViewBridge(private val mainActivity: MainActivity) { return com.google.ai.sample.util.TermuxExecutionModePreferences.executeInBackground(context) } + // ── Command Pattern Overrides (remote-updatable command syntax) ──────────── + // Lets the WebView bundle teach the native command parser new/alternate ways to spell + // an *existing* action (see CommandPatternConfig for the safety boundary). This is what + // makes "a new model emits slightly different command syntax" fixable via a repo commit + // instead of an app release. + + @JavascriptInterface + fun setCommandPatternOverrides(json: String): Int { + return try { + val applied = com.google.ai.sample.util.CommandParser.setRemotePatternOverrides(json) + com.google.ai.sample.util.CommandPatternOverridesPreferences.save(context, json) + applied + } catch (e: Exception) { + Log.e(TAG, "setCommandPatternOverrides error: ${e.message}") + 0 + } + } + + @JavascriptInterface + fun getCommandPatternOverrides(): String { + return com.google.ai.sample.util.CommandPatternOverridesPreferences.load(context) ?: "[]" + } + // ── Helpers ─────────────────────────────────────────────────────────────── companion object { diff --git a/app/src/main/kotlin/com/google/ai/sample/util/CommandParser.kt b/app/src/main/kotlin/com/google/ai/sample/util/CommandParser.kt index d1c458f6..1ec5e1bf 100644 --- a/app/src/main/kotlin/com/google/ai/sample/util/CommandParser.kt +++ b/app/src/main/kotlin/com/google/ai/sample/util/CommandParser.kt @@ -8,12 +8,19 @@ import android.util.Log object CommandParser { private const val TAG = "CommandParser" private val SINGLE_INSTANCE_COMMAND_TYPES = setOf( - CommandTypeEnum.TAKE_SCREENSHOT, - CommandTypeEnum.COMPLETED + CommandType.TAKE_SCREENSHOT, + CommandType.COMPLETED ) - // Enum to represent different command types - private enum class CommandTypeEnum { + /** + * Enum representing the different *kinds* of commands the app knows how to execute. + * + * This is intentionally public: [CommandPatternConfig] uses it to validate remotely + * supplied pattern overrides against a fixed whitelist, so that remote config can only + * ever attach a new regular expression to an action that already exists in compiled + * code - never introduce a brand-new kind of action. + */ + enum class CommandType { CLICK_BUTTON, LONG_CLICK_BUTTON, TAP_COORDINATES, TAKE_SCREENSHOT, COMPLETED, WAIT, PRESS_HOME, PRESS_BACK, SHOW_RECENT_APPS, SCROLL_DOWN, SCROLL_UP, SCROLL_LEFT, SCROLL_RIGHT, SCROLL_DOWN_FROM_COORDINATES, SCROLL_UP_FROM_COORDINATES, @@ -27,72 +34,116 @@ object CommandParser { val id: String, // For debugging val regex: Regex, val commandBuilder: (MatchResult) -> Command, - val commandType: CommandTypeEnum // Used for single-instance command check + val commandType: CommandType // Used for single-instance command check ) private data class ProcessedMatch( val startIndex: Int, val endIndex: Int, val command: Command, - val commandType: CommandTypeEnum + val commandType: CommandType ) // Master list of all patterns private val ALL_PATTERNS: List = listOf( // Enter key patterns - PatternInfo("enterKey1", Regex("(?i)\\benter\\(\\)"), { Command.PressEnterKey }, CommandTypeEnum.PRESS_ENTER_KEY), + PatternInfo("enterKey1", Regex("(?i)\\benter\\(\\)"), { Command.PressEnterKey }, CommandType.PRESS_ENTER_KEY), // Model selection patterns - PatternInfo("highReasoning1", Regex("(?i)\\bhighReasoningModel\\(\\)"), { Command.UseHighReasoningModel }, CommandTypeEnum.USE_HIGH_REASONING_MODEL), - PatternInfo("lowReasoning1", Regex("(?i)\\blowReasoningModel\\(\\)"), { Command.UseLowReasoningModel }, CommandTypeEnum.USE_LOW_REASONING_MODEL), + PatternInfo("highReasoning1", Regex("(?i)\\bhighReasoningModel\\(\\)"), { Command.UseHighReasoningModel }, CommandType.USE_HIGH_REASONING_MODEL), + PatternInfo("lowReasoning1", Regex("(?i)\\blowReasoningModel\\(\\)"), { Command.UseLowReasoningModel }, CommandType.USE_LOW_REASONING_MODEL), // Write text patterns - PatternInfo("writeText1", Regex("(?i)\\bwriteText\\([\"']([^\"']+)[\"']\\)"), { match -> Command.WriteText(match.groupValues[1]) }, CommandTypeEnum.WRITE_TEXT), - PatternInfo("termux1", Regex("""(?i)\bTermux\(\s*(["'])((?:\\.|(?!\1\s*\)).)*)\1\s*\)"""), { match -> Command.TermuxCommand(match.groupValues[2]) }, CommandTypeEnum.TERMUX_COMMAND), + PatternInfo("writeText1", Regex("(?i)\\bwriteText\\([\"']([^\"']+)[\"']\\)"), { match -> Command.WriteText(match.groupValues[1]) }, CommandType.WRITE_TEXT), + PatternInfo("termux1", Regex("""(?i)\bTermux\(\s*(["'])((?:\\.|(?!\1\s*\)).)*)\1\s*\)"""), { match -> Command.TermuxCommand(match.groupValues[2]) }, CommandType.TERMUX_COMMAND), // Click (long) button patterns - PatternInfo("clickBtn1", Regex("(?i)\\bclick\\([\"']([^\"']+)[\"']"), { match -> Command.ClickButton(match.groupValues[1]) }, CommandTypeEnum.CLICK_BUTTON), - PatternInfo("longClickBtn1", Regex("(?i)\\blongClick\\([\"']([^\"']+)[\"']"), { match -> Command.LongClickButton(match.groupValues[1]) }, CommandTypeEnum.LONG_CLICK_BUTTON), + PatternInfo("clickBtn1", Regex("(?i)\\bclick\\([\"']([^\"']+)[\"']"), { match -> Command.ClickButton(match.groupValues[1]) }, CommandType.CLICK_BUTTON), + PatternInfo("longClickBtn1", Regex("(?i)\\blongClick\\([\"']([^\"']+)[\"']"), { match -> Command.LongClickButton(match.groupValues[1]) }, CommandType.LONG_CLICK_BUTTON), // Tap coordinates patterns - PatternInfo("tapCoords1", Regex("(?i)\\btapAtCoordinates\\(\\s*([\\d\\.%]+)\\s*,\\s*([\\d\\.%]+)\\s*\\)"), { match -> Command.TapCoordinates(match.groupValues[1], match.groupValues[2]) }, CommandTypeEnum.TAP_COORDINATES), + PatternInfo("tapCoords1", Regex("(?i)\\btapAtCoordinates\\(\\s*([\\d\\.%]+)\\s*,\\s*([\\d\\.%]+)\\s*\\)"), { match -> Command.TapCoordinates(match.groupValues[1], match.groupValues[2]) }, CommandType.TAP_COORDINATES), // Screenshot, completion and wait patterns - PatternInfo("screenshot1", Regex("(?i)\\btakeScreenshot\\(\\)"), { Command.TakeScreenshot }, CommandTypeEnum.TAKE_SCREENSHOT), - PatternInfo("completed1", Regex("(?i)\\bcompleted\\(\\)"), { Command.Completed }, CommandTypeEnum.COMPLETED), - PatternInfo("wait1", Regex("(?i)\\bWait\\(\\s*(\\d+)\\s*\\)"), { match -> Command.Wait(match.groupValues[1].toLong()) }, CommandTypeEnum.WAIT), + PatternInfo("screenshot1", Regex("(?i)\\btakeScreenshot\\(\\)"), { Command.TakeScreenshot }, CommandType.TAKE_SCREENSHOT), + PatternInfo("completed1", Regex("(?i)\\bcompleted\\(\\)"), { Command.Completed }, CommandType.COMPLETED), + PatternInfo("wait1", Regex("(?i)\\bWait\\(\\s*(\\d+)\\s*\\)"), { match -> Command.Wait(match.groupValues[1].toLong()) }, CommandType.WAIT), // Home button patterns - PatternInfo("home1", Regex("(?i)\\bhome\\(\\)"), { Command.PressHomeButton }, CommandTypeEnum.PRESS_HOME), + PatternInfo("home1", Regex("(?i)\\bhome\\(\\)"), { Command.PressHomeButton }, CommandType.PRESS_HOME), // Back button patterns - PatternInfo("back1", Regex("(?i)\\bback\\(\\)"), { Command.PressBackButton }, CommandTypeEnum.PRESS_BACK), + PatternInfo("back1", Regex("(?i)\\bback\\(\\)"), { Command.PressBackButton }, CommandType.PRESS_BACK), // Recent apps patterns - PatternInfo("recentApps1", Regex("(?i)\\brecentApps\\(\\)"), { Command.ShowRecentApps }, CommandTypeEnum.SHOW_RECENT_APPS), + PatternInfo("recentApps1", Regex("(?i)\\brecentApps\\(\\)"), { Command.ShowRecentApps }, CommandType.SHOW_RECENT_APPS), // Scroll patterns (simple) - PatternInfo("scrollDown1", Regex("(?i)\\bscrollDown\\(\\)"), { Command.ScrollDown }, CommandTypeEnum.SCROLL_DOWN), - PatternInfo("scrollUp1", Regex("(?i)\\bscrollUp\\(\\)"), { Command.ScrollUp }, CommandTypeEnum.SCROLL_UP), - PatternInfo("scrollLeft1", Regex("(?i)\\bscrollLeft\\(\\)"), { Command.ScrollLeft }, CommandTypeEnum.SCROLL_LEFT), - PatternInfo("scrollRight1", Regex("(?i)\\bscrollRight\\(\\)"), { Command.ScrollRight }, CommandTypeEnum.SCROLL_RIGHT), + PatternInfo("scrollDown1", Regex("(?i)\\bscrollDown\\(\\)"), { Command.ScrollDown }, CommandType.SCROLL_DOWN), + PatternInfo("scrollUp1", Regex("(?i)\\bscrollUp\\(\\)"), { Command.ScrollUp }, CommandType.SCROLL_UP), + PatternInfo("scrollLeft1", Regex("(?i)\\bscrollLeft\\(\\)"), { Command.ScrollLeft }, CommandType.SCROLL_LEFT), + PatternInfo("scrollRight1", Regex("(?i)\\bscrollRight\\(\\)"), { Command.ScrollRight }, CommandType.SCROLL_RIGHT), // Scroll from coordinates patterns PatternInfo("scrollDownCoords", Regex("(?i)\\bscrollDown\\s*\\(\\s*([\\d\\.%]+)\\s*,\\s*([\\d\\.%]+)\\s*,\\s*([\\d\\.%]+)\\s*,\\s*(\\d+)\\s*\\)"), - { match -> Command.ScrollDownFromCoordinates(match.groupValues[1], match.groupValues[2], match.groupValues[3], match.groupValues[4].toLong()) }, CommandTypeEnum.SCROLL_DOWN_FROM_COORDINATES), + { match -> Command.ScrollDownFromCoordinates(match.groupValues[1], match.groupValues[2], match.groupValues[3], match.groupValues[4].toLong()) }, CommandType.SCROLL_DOWN_FROM_COORDINATES), PatternInfo("scrollUpCoords", Regex("(?i)\\bscrollUp\\s*\\(\\s*([\\d\\.%]+)\\s*,\\s*([\\d\\.%]+)\\s*,\\s*([\\d\\.%]+)\\s*,\\s*(\\d+)\\s*\\)"), - { match -> Command.ScrollUpFromCoordinates(match.groupValues[1], match.groupValues[2], match.groupValues[3], match.groupValues[4].toLong()) }, CommandTypeEnum.SCROLL_UP_FROM_COORDINATES), + { match -> Command.ScrollUpFromCoordinates(match.groupValues[1], match.groupValues[2], match.groupValues[3], match.groupValues[4].toLong()) }, CommandType.SCROLL_UP_FROM_COORDINATES), PatternInfo("scrollLeftCoords", Regex("(?i)\\bscrollLeft\\s*\\(\\s*([\\d\\.%]+)\\s*,\\s*([\\d\\.%]+)\\s*,\\s*([\\d\\.%]+)\\s*,\\s*(\\d+)\\s*\\)"), - { match -> Command.ScrollLeftFromCoordinates(match.groupValues[1], match.groupValues[2], match.groupValues[3], match.groupValues[4].toLong()) }, CommandTypeEnum.SCROLL_LEFT_FROM_COORDINATES), + { match -> Command.ScrollLeftFromCoordinates(match.groupValues[1], match.groupValues[2], match.groupValues[3], match.groupValues[4].toLong()) }, CommandType.SCROLL_LEFT_FROM_COORDINATES), PatternInfo("scrollRightCoords", Regex("(?i)\\bscrollRight\\s*\\(\\s*([\\d\\.%]+)\\s*,\\s*([\\d\\.%]+)\\s*,\\s*([\\d\\.%]+)\\s*,\\s*(\\d+)\\s*\\)"), - { match -> Command.ScrollRightFromCoordinates(match.groupValues[1], match.groupValues[2], match.groupValues[3], match.groupValues[4].toLong()) }, CommandTypeEnum.SCROLL_RIGHT_FROM_COORDINATES), + { match -> Command.ScrollRightFromCoordinates(match.groupValues[1], match.groupValues[2], match.groupValues[3], match.groupValues[4].toLong()) }, CommandType.SCROLL_RIGHT_FROM_COORDINATES), // Open app patterns - PatternInfo("openApp1", Regex("(?i)\\bopenApp\\([\"']([^\"']+)[\"']\\)"), { match -> Command.OpenApp(match.groupValues[1]) }, CommandTypeEnum.OPEN_APP), + PatternInfo("openApp1", Regex("(?i)\\bopenApp\\([\"']([^\"']+)[\"']\\)"), { match -> Command.OpenApp(match.groupValues[1]) }, CommandType.OPEN_APP), // Retrieve information patterns - PatternInfo("retrieve1", Regex("(?i)\\bretrieve\\([\"']([^\"']+)[\"']\\)"), { match -> Command.Retrieve(match.groupValues[1]) }, CommandTypeEnum.RETRIEVE) + PatternInfo("retrieve1", Regex("(?i)\\bretrieve\\([\"']([^\"']+)[\"']\\)"), { match -> Command.Retrieve(match.groupValues[1]) }, CommandType.RETRIEVE) ) + // One canonical command-builder per CommandType, derived from ALL_PATTERNS above. + // CommandPatternConfig uses this to attach remotely supplied regexes to the existing, + // compiled-in command-construction logic - it can never introduce custom logic of its own. + private val BUILDER_BY_TYPE: Map Command> by lazy { + ALL_PATTERNS.associate { it.commandType to it.commandBuilder } + } + + // Additional patterns supplied at runtime (e.g. fetched together with the WebView bundle) + // so that a new model's slightly different command syntax can be recognized without an + // app update. See [CommandPatternConfig] for the safety boundary this respects. Empty by + // default, i.e. behavior is unchanged unless overrides are explicitly installed. + @Volatile + private var remotePatterns: List = emptyList() + + /** + * Installs additional command-recognition patterns from a remotely supplied JSON config. + * Each entry may only reference an existing [CommandType]; unknown types or invalid + * regexes are skipped (logged) rather than causing a crash, so a malformed remote config + * degrades gracefully to "no extra patterns". + * + * @return the number of overrides that were successfully installed. + */ + @Synchronized + fun setRemotePatternOverrides(json: String): Int { + val parsed = CommandPatternConfig.parse(json) + remotePatterns = parsed.mapNotNull { override -> + val builder = BUILDER_BY_TYPE[override.commandType] + if (builder == null) { + Log.w(TAG, "Skipping remote pattern override '${override.id}': no builder for ${override.commandType}") + null + } else { + PatternInfo(override.id, override.regex, builder, override.commandType) + } + } + Log.d(TAG, "Installed ${remotePatterns.size} remote command pattern override(s)") + return remotePatterns.size + } + + /** Removes all remotely installed pattern overrides, reverting to built-in patterns only. */ + @Synchronized + fun clearRemotePatternOverrides() { + remotePatterns = emptyList() + } + // Buffer for storing partial text between calls private var textBuffer = "" @@ -184,7 +235,7 @@ object CommandParser { private fun processTextInternal(text: String): List { val foundRawMatches = collectRawMatches(text) val finalCommands = mutableListOf() - val addedSingleInstanceCommands = mutableSetOf() + val addedSingleInstanceCommands = mutableSetOf() // Sort matches by start index foundRawMatches.sortBy { it.startIndex } @@ -213,7 +264,7 @@ object CommandParser { private fun collectRawMatches(text: String): MutableList { val foundRawMatches = mutableListOf() - for (patternInfo in ALL_PATTERNS) { + for (patternInfo in ALL_PATTERNS + remotePatterns) { try { patternInfo.regex.findAll(text).forEach { matchResult -> try { diff --git a/app/src/main/kotlin/com/google/ai/sample/util/CommandPatternConfig.kt b/app/src/main/kotlin/com/google/ai/sample/util/CommandPatternConfig.kt new file mode 100644 index 00000000..143274d4 --- /dev/null +++ b/app/src/main/kotlin/com/google/ai/sample/util/CommandPatternConfig.kt @@ -0,0 +1,85 @@ +package com.google.ai.sample.util + +import android.util.Log +import org.json.JSONArray + +/** + * Allows the set of recognized command *syntaxes* to be extended at runtime from a remotely + * fetched JSON config (e.g. shipped alongside the WebView's index.html on the + * feature/webview-test branch), without requiring a new app release. + * + * Example payload (a JSON array, e.g. "command-patterns.json" next to index.html): + * ```json + * [ + * { "id": "clickBtnCapitalized", "commandType": "CLICK_BUTTON", "regex": "(?i)\\bClick\\([\"']([^\"']+)[\"']" } + * ] + * ``` + * + * IMPORTANT (safety boundary): an override can only attach a *new regular expression* to an + * *already existing* [CommandParser.CommandType]. It can never introduce a new kind of action, + * and it can never run arbitrary code - the actual [Command] that gets built (and therefore + * everything that is allowed to happen on the device) is still produced by the same, + * compiled-in builder function that ships with the app for that command type. This means a new + * model that simply phrases an existing action differently (e.g. "Click('...')" instead of + * "click(\"...\")") can be supported purely by editing a JSON file in the repo - while what each + * action is actually allowed to do stays fixed in native code and unrelated to this mechanism. + */ +internal object CommandPatternConfig { + private const val TAG = "CommandPatternConfig" + + data class ParsedOverride( + val id: String, + val commandType: CommandParser.CommandType, + val regex: Regex + ) + + /** + * Parses a JSON array of pattern overrides. Any entry that is malformed, references an + * unknown command type, or contains an invalid regex is skipped (and logged) instead of + * throwing, so a bad remote config degrades to "no extra patterns" rather than crashing + * the app or blocking recognition of built-in patterns. + */ + fun parse(json: String): List { + val result = mutableListOf() + if (json.isBlank()) return result + + try { + val array = JSONArray(json) + for (i in 0 until array.length()) { + val entry = array.optJSONObject(i) + if (entry == null) { + Log.w(TAG, "Skipping override at index $i: not a JSON object") + continue + } + + val id = entry.optString("id", "remote_$i") + val typeName = entry.optString("commandType", "") + val pattern = entry.optString("regex", "") + + if (pattern.isBlank()) { + Log.w(TAG, "Skipping override '$id': empty/missing regex") + continue + } + + val commandType = try { + CommandParser.CommandType.valueOf(typeName) + } catch (e: IllegalArgumentException) { + Log.w(TAG, "Skipping override '$id': unknown commandType '$typeName'") + continue + } + + val regex = try { + Regex(pattern) + } catch (e: Exception) { + Log.w(TAG, "Skipping override '$id': invalid regex '$pattern' (${e.message})") + continue + } + + result.add(ParsedOverride(id, commandType, regex)) + } + } catch (e: Exception) { + Log.e(TAG, "Failed to parse remote command pattern overrides: ${e.message}", e) + } + return result + } +} diff --git a/app/src/main/kotlin/com/google/ai/sample/util/CommandPatternOverridesPreferences.kt b/app/src/main/kotlin/com/google/ai/sample/util/CommandPatternOverridesPreferences.kt new file mode 100644 index 00000000..b03ef6cd --- /dev/null +++ b/app/src/main/kotlin/com/google/ai/sample/util/CommandPatternOverridesPreferences.kt @@ -0,0 +1,38 @@ +package com.google.ai.sample.util + +import android.content.Context +import android.util.Log +import androidx.core.content.edit + +/** + * Persists the most recently received remote command-pattern override JSON (see + * [CommandPatternConfig] / [CommandParser.setRemotePatternOverrides]) so that recognition of + * additional/alternate command syntaxes keeps working across app restarts - including before + * the WebView bundle has re-fetched and re-applied it for the current session. + */ +object CommandPatternOverridesPreferences { + private const val TAG = "CmdPatternOverridesPrefs" + private const val PREFS_NAME = "command_pattern_overrides_prefs" + private const val KEY_JSON = "overrides_json" + + private fun prefs(context: Context) = context.getSharedPreferences(PREFS_NAME, Context.MODE_PRIVATE) + + /** Saves the raw override JSON as last received from the WebView/remote bundle. */ + fun save(context: Context, json: String) { + try { + prefs(context).edit { putString(KEY_JSON, json) } + } catch (e: Exception) { + Log.e(TAG, "Error saving command pattern overrides: ${e.message}", e) + } + } + + /** Loads the last saved override JSON, or null if none has been received yet. */ + fun load(context: Context): String? { + return try { + prefs(context).getString(KEY_JSON, null) + } catch (e: Exception) { + Log.e(TAG, "Error loading command pattern overrides: ${e.message}", e) + null + } + } +} diff --git a/app/src/test/java/com/google/ai/sample/util/CommandParserTest.kt b/app/src/test/java/com/google/ai/sample/util/CommandParserTest.kt index 91fe6447..c193f99b 100644 --- a/app/src/test/java/com/google/ai/sample/util/CommandParserTest.kt +++ b/app/src/test/java/com/google/ai/sample/util/CommandParserTest.kt @@ -10,6 +10,7 @@ class CommandParserTest { @Before fun setUp() { CommandParser.clearBuffer() + CommandParser.clearRemotePatternOverrides() } @Test @@ -123,4 +124,50 @@ class CommandParserTest { assertEquals("su -c \"ifconfig\"", (command as Command.TermuxCommand).command) } + @Test + fun setRemotePatternOverrides_recognizesAlternateSyntaxForExistingCommandType() { + // A hypothetical new model emits "Click('...')" (capitalized, single quotes) instead + // of the built-in "click(\"...\")" syntax. Without an override, this is NOT recognized: + val before = CommandParser.parseCommands("Click('Login')", clearBuffer = true) + assertEquals(0, before.size) + + val applied = CommandParser.setRemotePatternOverrides( + """[{"id":"clickAlt","commandType":"CLICK_BUTTON","regex":"(?i)\\bClick\\([\"']([^\"']+)[\"']"}]""" + ) + assertEquals(1, applied) + + val after = CommandParser.parseCommands("Click('Login')", clearBuffer = true) + assertEquals(1, after.size) + assertTrue(after.first() is Command.ClickButton) + assertEquals("Login", (after.first() as Command.ClickButton).buttonText) + } + + @Test + fun setRemotePatternOverrides_skipsUnknownCommandType() { + val applied = CommandParser.setRemotePatternOverrides( + """[{"id":"bogus","commandType":"DOES_NOT_EXIST","regex":"(?i)\\bfoo\\(\\)"}]""" + ) + assertEquals(0, applied) + } + + @Test + fun setRemotePatternOverrides_skipsInvalidRegexWithoutCrashing() { + val applied = CommandParser.setRemotePatternOverrides( + """[{"id":"badRegex","commandType":"CLICK_BUTTON","regex":"("}]""" + ) + assertEquals(0, applied) + } + + @Test + fun clearRemotePatternOverrides_revertsToBuiltInPatternsOnly() { + CommandParser.setRemotePatternOverrides( + """[{"id":"clickAlt","commandType":"CLICK_BUTTON","regex":"(?i)\\bClick\\([\"']([^\"']+)[\"']"}]""" + ) + assertEquals(1, CommandParser.parseCommands("Click('Login')", clearBuffer = true).size) + + CommandParser.clearRemotePatternOverrides() + + assertEquals(0, CommandParser.parseCommands("Click('Login')", clearBuffer = true).size) + } + } diff --git a/command-patterns.json b/command-patterns.json new file mode 100644 index 00000000..fe51488c --- /dev/null +++ b/command-patterns.json @@ -0,0 +1 @@ +[] diff --git a/docs/command-pattern-overrides.md b/docs/command-pattern-overrides.md new file mode 100644 index 00000000..7b9bd502 --- /dev/null +++ b/docs/command-pattern-overrides.md @@ -0,0 +1,64 @@ +# Command pattern overrides (remote-updatable command syntax) + +`CommandParser.kt` recognizes the action commands an AI model emits (`click("...")`, +`tapAtCoordinates(x, y)`, `scrollDown()`, ...) using a fixed set of built-in regular +expressions. Until now, supporting a new model that phrases an existing action slightly +differently (e.g. `Click('...')` instead of `click("...")`) required patching +`CommandParser.kt` and shipping a new app release. + +`command-patterns.json` (this file, fetched by the WebView relative to `index.html`) lets +you add such alternate spellings without an app release. It is optional — if the file is +missing or invalid, the app silently falls back to the built-in patterns only. + +## Format + +A JSON array of override objects: + +```json +[ + { + "id": "clickBtnCapitalized", + "commandType": "CLICK_BUTTON", + "regex": "(?i)\\bClick\\([\"']([^\"']+)[\"']" + } +] +``` + +- `id` — any string, used only for logging. +- `commandType` — must be one of the values below. Unknown values are skipped (logged), + not an error. +- `regex` — a Kotlin/Java regular expression. **It must capture the same groups, in the + same order, as the built-in pattern for that `commandType`** (see `CommandParser.kt`), + since the existing, compiled-in builder function reads `match.groupValues[...]` to + construct the command. If the group count doesn't match, that particular match is + skipped (logged), nothing crashes. + +## Safety boundary + +An override can only attach a new regex to an **existing** `commandType` — it can never +introduce a new kind of action and can never run custom code. What each action is allowed +to do on the device (tap, scroll, open an app, run a Termux command, ...) is always +decided by the same native, compiled-in code; this mechanism only changes which *text* +triggers that pre-existing action. Adding a genuinely new action still requires a native +code change. + +## Available `commandType` values + +`CLICK_BUTTON`, `LONG_CLICK_BUTTON`, `TAP_COORDINATES`, `TAKE_SCREENSHOT`, `COMPLETED`, +`WAIT`, `PRESS_HOME`, `PRESS_BACK`, `SHOW_RECENT_APPS`, `SCROLL_DOWN`, `SCROLL_UP`, +`SCROLL_LEFT`, `SCROLL_RIGHT`, `SCROLL_DOWN_FROM_COORDINATES`, `SCROLL_UP_FROM_COORDINATES`, +`SCROLL_LEFT_FROM_COORDINATES`, `SCROLL_RIGHT_FROM_COORDINATES`, `OPEN_APP`, `WRITE_TEXT`, +`USE_HIGH_REASONING_MODEL`, `USE_LOW_REASONING_MODEL`, `PRESS_ENTER_KEY`, `RETRIEVE`, +`TERMUX_COMMAND`. + +## How it gets applied + +1. The WebView loads `index.html` and fires `window.onAndroidReady()`. +2. That handler fetches `command-patterns.json` (relative to the WebView's base URL) and + passes its raw text to `Android.setCommandPatternOverrides(json)`. +3. The native `CommandParser` installs the parsed overrides and the bridge persists the + raw JSON via `CommandPatternOverridesPreferences`, so it's restored on the next app + start (in `PhotoReasoningApplication.onCreate()`) even before the WebView reloads it. + +To add support for a new model's command syntax: edit `command-patterns.json` in this +repo and commit — no new app version needed. diff --git a/index.html b/index.html index 4296ca55..38d383be 100644 --- a/index.html +++ b/index.html @@ -506,6 +506,10 @@ /* Donation / trial */ initiateDonation:()=>{ if(getInAndroid()) Android.initiateDonation(); else alert('Subscription flow would open here'); }, isPurchased:()=> getInAndroid() ? Android.isPurchased() : false, + + /* Command pattern overrides (remote-updatable command syntax for new models) */ + setCommandPatternOverrides:(json)=>{ if(getInAndroid()) return Android.setCommandPatternOverrides(json); }, + getCommandPatternOverrides:()=> getInAndroid() ? Android.getCommandPatternOverrides() : '[]', }; /* ════════════════════════════════════════════════════════ @@ -585,6 +589,21 @@ let termuxBackground = false; let generationRunning = false; +/* ════════════════════════════════════════════════════════ + COMMAND PATTERN OVERRIDES + Optional file "command-patterns.json" next to this index.html, fetched relative to + the WebView's base URL. Lets a new model's slightly different command syntax (e.g. + "Click('...')" instead of "click(\"...\")") be supported via a repo commit, without + an app update. Safe to omit entirely - 404/parse errors are ignored. +════════════════════════════════════════════════════════ */ +window.onAndroidReady = function() { + if (!getInAndroid()) return; + fetch('command-patterns.json', { cache: 'no-store' }) + .then(r => r.ok ? r.text() : null) + .then(json => { if (json) Bridge.setCommandPatternOverrides(json); }) + .catch(() => { /* file is optional; ignore if missing or fetch fails */ }); +}; + /* ════════════════════════════════════════════════════════ INIT ════════════════════════════════════════════════════════ */ From 924511403e4e553e6814d4c21ee0d080b4f49e90 Mon Sep 17 00:00:00 2001 From: Claude Date: Fri, 26 Jun 2026 04:55:03 +0000 Subject: [PATCH 06/11] Add custom, fully JSON-defined models with the API call made in JS (Punkt 2) Lets a genuinely new model/provider be added with zero app release: define it in custom-models.json (endpoint, modelName, auth header) and the actual HTTP request is made by JavaScript directly in the WebView (window.onCustomModelRequest, fetch()), not by native networking code. Requires the provider's endpoint to support CORS for browser-style requests - verify this per provider, it is not guaranteed. Native side (additive only, zero changes to any existing ModelOption's behavior): - CustomModelDefinition/CustomModelConfig: data class + JSON parser for custom-models.json, completely separate from ModelOption/GenerativeAiViewModelFactory. - CustomModelRegistry: in-memory active list + active selection, independent of the ModelOption enum. - CustomModelPreferences: persists the models json, the active selection, and a per-model API key (custom models aren't tied to the existing ApiProvider enum/ ApiKeyManager storage). - WebViewBridge.setSelectedModel(id) now falls back to CustomModelRegistry when id isn't a ModelOption - this is the minimal slice of 'decouple model selection from the enum' (Punkt 1) needed for Punkt 2 to be selectable at all. Removed a dead, orphaned addCustomModel() no-op stub from an earlier, abandoned attempt at this. - PhotoReasoningViewModel.reason(): if a custom model is active, delegates to the new reasonWithCustomJsModel(), which builds the request context (system message, db entries, sanitized history, user text, base64 images) and emits it on the new customModelRequestEvents SharedFlow instead of calling any provider itself. - onCustomModelPartialResponse/onCustomModelFinalResponse/onCustomModelError: new public ViewModel methods, called from WebViewBridge once JS has the result. They reuse the exact same replaceAiMessageText/processCommandsIncrementally/ finalizeAiMessage/processCommands/saveChatHistory pipeline every other model already uses, so command execution and persistence behave identically. WebView side (index.html): - custom-models.json is fetched on window.onAndroidReady() (merged the fetch into the pre-existing onAndroidReady - there were two conflicting definitions of it before this commit, the second silently overwriting the first; fixed as part of this change) and merged into the MODELS array / model picker. - window.onCustomModelRequest(payloadJson): builds an OpenAI-compatible chat- completions request, calls fetch(), and either parses SSE streaming chunks or a single JSON response, reporting back via the three bridge callbacks above. - stopGeneration() now also aborts an in-flight custom-model fetch() via AbortController, so Stop works the same way regardless of which model is active. Docs: docs/custom-models.md (format, API key setup, request flow, explicit limitations: CORS must be verified per provider; generation settings sliders are not yet persisted per custom model; the model's API key is necessarily visible to JS to set the auth header, consistent with the existing getAllApiKeys() exposure). Tests: CustomModelConfigTest, CustomModelRegistryTest (pure JVM, no Android context needed). Verified index.html's extracted