diff options
Diffstat (limited to 'java/com/android/voicemail/impl/transcribe')
8 files changed, 919 insertions, 0 deletions
diff --git a/java/com/android/voicemail/impl/transcribe/TranscriptionConfigProvider.java b/java/com/android/voicemail/impl/transcribe/TranscriptionConfigProvider.java new file mode 100644 index 000000000..17c9be73b --- /dev/null +++ b/java/com/android/voicemail/impl/transcribe/TranscriptionConfigProvider.java @@ -0,0 +1,62 @@ +/* + * Copyright (C) 2017 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License + */ +package com.android.voicemail.impl.transcribe; + +import android.content.Context; +import com.android.dialer.common.ConfigProviderBindings; + +/** Provides configuration values needed to connect to the transcription server. */ +public class TranscriptionConfigProvider { + private final Context context; + + public TranscriptionConfigProvider(Context context) { + this.context = context; + } + + public boolean isVoicemailTranscriptionEnabled() { + return ConfigProviderBindings.get(context).getBoolean("voicemail_transcription_enabled", false); + } + + public String getServerAddress() { + // Private voicemail transcription service + return ConfigProviderBindings.get(context) + .getString( + "voicemail_transcription_server_address", "voicemailtranscription-pa.googleapis.com"); + } + + public String getApiKey() { + // Android API key restricted to com.google.android.dialer + return ConfigProviderBindings.get(context) + .getString( + "voicemail_transcription_client_api_key", "AIzaSyAXdDnif6B7sBYxU8hzw9qAp3pRPVHs060"); + } + + public String getAuthToken() { + return null; + } + + public boolean shouldUsePlaintext() { + return ConfigProviderBindings.get(context) + .getBoolean("voicemail_transcription_server_use_plaintext", false); + } + + @Override + public String toString() { + return String.format( + "{ address: %s, api key: %s, auth token: %s, plaintext: %b }", + getServerAddress(), getApiKey(), getAuthToken(), shouldUsePlaintext()); + } +} diff --git a/java/com/android/voicemail/impl/transcribe/TranscriptionDbHelper.java b/java/com/android/voicemail/impl/transcribe/TranscriptionDbHelper.java new file mode 100644 index 000000000..cbc5cb8a0 --- /dev/null +++ b/java/com/android/voicemail/impl/transcribe/TranscriptionDbHelper.java @@ -0,0 +1,105 @@ +/* + * Copyright (C) 2017 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License + */ +package com.android.voicemail.impl.transcribe; + +import android.annotation.TargetApi; +import android.content.ContentResolver; +import android.content.ContentValues; +import android.content.Context; +import android.database.Cursor; +import android.net.Uri; +import android.os.Build.VERSION_CODES; +import android.provider.VoicemailContract.Voicemails; +import android.support.annotation.WorkerThread; +import android.support.v4.os.BuildCompat; +import android.util.Pair; +import com.android.dialer.common.Assert; +import com.android.dialer.common.LogUtil; + +/** Helper class for reading and writing transcription data in the database */ +@TargetApi(VERSION_CODES.O) +public class TranscriptionDbHelper { + private static final String[] PROJECTION = + new String[] { + Voicemails.TRANSCRIPTION, // 0 + VoicemailCompat.TRANSCRIPTION_STATE // 1 + }; + + public static final int TRANSCRIPTION = 0; + public static final int TRANSCRIPTION_STATE = 1; + + private final ContentResolver contentResolver; + private final Uri uri; + + TranscriptionDbHelper(Context context, Uri uri) { + Assert.isNotNull(uri); + this.contentResolver = context.getContentResolver(); + this.uri = uri; + } + + @WorkerThread + @TargetApi(VERSION_CODES.M) // used for try with resources + Pair<String, Integer> getTranscriptionAndState() { + Assert.checkArgument(BuildCompat.isAtLeastO()); + Assert.isWorkerThread(); + try (Cursor cursor = contentResolver.query(uri, PROJECTION, null, null, null)) { + if (cursor == null) { + LogUtil.e("TranscriptionDbHelper.getTranscriptionAndState", "query failed."); + return null; + } + + if (cursor.moveToFirst()) { + String transcription = cursor.getString(TRANSCRIPTION); + int transcriptionState = cursor.getInt(TRANSCRIPTION_STATE); + return new Pair<>(transcription, transcriptionState); + } + } + LogUtil.i("TranscriptionDbHelper.getTranscriptionAndState", "query returned no results"); + return null; + } + + @WorkerThread + void setTranscriptionState(int transcriptionState) { + Assert.isWorkerThread(); + LogUtil.i( + "TranscriptionDbHelper.setTranscriptionState", + "uri: " + uri + ", state: " + transcriptionState); + ContentValues values = new ContentValues(); + values.put(VoicemailCompat.TRANSCRIPTION_STATE, transcriptionState); + updateDatabase(values); + } + + @WorkerThread + void setTranscriptionAndState(String transcription, int transcriptionState) { + Assert.isWorkerThread(); + LogUtil.i( + "TranscriptionDbHelper.setTranscriptionAndState", + "uri: " + uri + ", state: " + transcriptionState); + ContentValues values = new ContentValues(); + values.put(Voicemails.TRANSCRIPTION, transcription); + values.put(VoicemailCompat.TRANSCRIPTION_STATE, transcriptionState); + updateDatabase(values); + } + + private void updateDatabase(ContentValues values) { + int updatedCount = contentResolver.update(uri, values, null, null); + if (updatedCount != 1) { + LogUtil.e( + "TranscriptionDbHelper.updateDatabase", + "Wrong row count, should have updated 1 row, was: " + updatedCount); + } + } +} diff --git a/java/com/android/voicemail/impl/transcribe/TranscriptionService.java b/java/com/android/voicemail/impl/transcribe/TranscriptionService.java new file mode 100644 index 000000000..3e80a7f59 --- /dev/null +++ b/java/com/android/voicemail/impl/transcribe/TranscriptionService.java @@ -0,0 +1,203 @@ +/* + * Copyright (C) 2017 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License + */ +package com.android.voicemail.impl.transcribe; + +import android.app.job.JobInfo; +import android.app.job.JobParameters; +import android.app.job.JobScheduler; +import android.app.job.JobService; +import android.app.job.JobWorkItem; +import android.content.ComponentName; +import android.content.Context; +import android.content.Intent; +import android.net.Uri; +import android.os.StrictMode; +import android.support.annotation.MainThread; +import android.support.annotation.VisibleForTesting; +import android.support.v4.os.BuildCompat; +import android.text.TextUtils; +import com.android.dialer.common.Assert; +import com.android.dialer.common.LogUtil; +import com.android.dialer.constants.ScheduledJobIds; +import com.android.voicemail.impl.transcribe.grpc.TranscriptionClientFactory; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; + +/** + * Job scheduler callback for launching voicemail transcription tasks. The transcription tasks will + * run in the background and will typically last for approximately the length of the voicemail audio + * (since thats how long the backend transcription service takes to do the transcription). + */ +public class TranscriptionService extends JobService { + @VisibleForTesting static final String EXTRA_VOICEMAIL_URI = "extra_voicemail_uri"; + + private ExecutorService executorService; + private JobParameters jobParameters; + private TranscriptionClientFactory clientFactory; + private TranscriptionConfigProvider configProvider; + private StrictMode.VmPolicy originalPolicy; + + /** Callback used by a task to indicate it has finished processing its work item */ + interface JobCallback { + void onWorkCompleted(JobWorkItem completedWorkItem); + } + + // Schedule a task to transcribe the indicated voicemail, return true if transcription task was + // scheduled. + public static boolean transcribeVoicemail(Context context, Uri voicemailUri) { + Assert.isMainThread(); + if (BuildCompat.isAtLeastO()) { + LogUtil.i("TranscriptionService.transcribeVoicemail", "scheduling transcription"); + ComponentName componentName = new ComponentName(context, TranscriptionService.class); + JobInfo.Builder builder = + new JobInfo.Builder(ScheduledJobIds.VVM_TRANSCRIPTION_JOB, componentName) + .setMinimumLatency(0) + .setRequiredNetworkType(JobInfo.NETWORK_TYPE_ANY); + JobScheduler scheduler = context.getSystemService(JobScheduler.class); + JobWorkItem workItem = makeWorkItem(voicemailUri); + return scheduler.enqueue(builder.build(), workItem) == JobScheduler.RESULT_SUCCESS; + } else { + LogUtil.i("TranscriptionService.transcribeVoicemail", "not supported"); + return false; + } + } + + // Cancel all transcription tasks + public static void cancelTranscriptions(Context context) { + Assert.isMainThread(); + LogUtil.enterBlock("TranscriptionService.cancelTranscriptions"); + JobScheduler scheduler = context.getSystemService(JobScheduler.class); + scheduler.cancel(ScheduledJobIds.VVM_TRANSCRIPTION_JOB); + } + + public TranscriptionService() { + Assert.isMainThread(); + } + + @VisibleForTesting + TranscriptionService( + ExecutorService executorService, + TranscriptionClientFactory clientFactory, + TranscriptionConfigProvider configProvider) { + this.executorService = executorService; + this.clientFactory = clientFactory; + this.configProvider = configProvider; + } + + @Override + public boolean onStartJob(JobParameters params) { + Assert.isMainThread(); + LogUtil.enterBlock("TranscriptionService.onStartJob"); + if (!getConfigProvider().isVoicemailTranscriptionEnabled()) { + LogUtil.i("TranscriptionService.onStartJob", "transcription not enabled, exiting."); + return false; + } else if (TextUtils.isEmpty(getConfigProvider().getServerAddress())) { + LogUtil.i("TranscriptionService.onStartJob", "transcription server not configured, exiting."); + return false; + } else { + LogUtil.i( + "TranscriptionService.onStartJob", + "transcription server address: " + configProvider.getServerAddress()); + originalPolicy = StrictMode.getVmPolicy(); + StrictMode.enableDefaults(); + jobParameters = params; + return checkForWork(); + } + } + + @Override + public boolean onStopJob(JobParameters params) { + Assert.isMainThread(); + LogUtil.enterBlock("TranscriptionService.onStopJob"); + cleanup(); + return true; + } + + @Override + public void onDestroy() { + Assert.isMainThread(); + LogUtil.enterBlock("TranscriptionService.onDestroy"); + cleanup(); + } + + private void cleanup() { + if (clientFactory != null) { + clientFactory.shutdown(); + clientFactory = null; + } + if (executorService != null) { + executorService.shutdownNow(); + executorService = null; + } + if (originalPolicy != null) { + StrictMode.setVmPolicy(originalPolicy); + originalPolicy = null; + } + } + + @MainThread + private boolean checkForWork() { + Assert.isMainThread(); + JobWorkItem workItem = jobParameters.dequeueWork(); + if (workItem != null) { + getExecutorService() + .execute(new TranscriptionTask(this, new Callback(), workItem, getClientFactory())); + return true; + } else { + return false; + } + } + + private ExecutorService getExecutorService() { + if (executorService == null) { + // The common use case is transcribing a single voicemail so just use a single thread executor + // The reason we're not using DialerExecutor here is because the transcription task can be + // very long running (ie. multiple minutes). + executorService = Executors.newSingleThreadExecutor(); + } + return executorService; + } + + private class Callback implements JobCallback { + @Override + public void onWorkCompleted(JobWorkItem completedWorkItem) { + Assert.isMainThread(); + LogUtil.i("TranscriptionService.Callback.onWorkCompleted", completedWorkItem.toString()); + jobParameters.completeWork(completedWorkItem); + checkForWork(); + } + } + + private static JobWorkItem makeWorkItem(Uri voicemailUri) { + Intent intent = new Intent(); + intent.putExtra(EXTRA_VOICEMAIL_URI, voicemailUri); + return new JobWorkItem(intent); + } + + private TranscriptionConfigProvider getConfigProvider() { + if (configProvider == null) { + configProvider = new TranscriptionConfigProvider(this); + } + return configProvider; + } + + private TranscriptionClientFactory getClientFactory() { + if (clientFactory == null) { + clientFactory = new TranscriptionClientFactory(this, getConfigProvider()); + } + return clientFactory; + } +} diff --git a/java/com/android/voicemail/impl/transcribe/TranscriptionTask.java b/java/com/android/voicemail/impl/transcribe/TranscriptionTask.java new file mode 100644 index 000000000..0fbc33ad5 --- /dev/null +++ b/java/com/android/voicemail/impl/transcribe/TranscriptionTask.java @@ -0,0 +1,191 @@ +/* + * Copyright (C) 2017 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License + */ +package com.android.voicemail.impl.transcribe; + +import android.annotation.TargetApi; +import android.app.job.JobWorkItem; +import android.content.Context; +import android.net.Uri; +import android.text.TextUtils; +import com.android.dialer.common.concurrent.ThreadUtil; +import com.android.voicemail.impl.VvmLog; +import com.android.voicemail.impl.transcribe.TranscriptionService.JobCallback; +import com.android.voicemail.impl.transcribe.grpc.TranscriptionClient; +import com.android.voicemail.impl.transcribe.grpc.TranscriptionClientFactory; +import com.google.internal.communications.voicemailtranscription.v1.AudioFormat; +import com.google.internal.communications.voicemailtranscription.v1.TranscribeVoicemailRequest; +import com.google.protobuf.ByteString; +import io.grpc.Status; +import java.io.IOException; +import java.io.InputStream; + +/** + * Background task to get a voicemail transcription and update the database. + * + * <pre> + * This task performs the following steps: + * 1. Update the transcription-state in the database to 'in-progress' + * 2. Create grpc client and transcription request + * 3. Make synchronous grpc transcription request to backend server + * 3a. On response + * Update the database with transcription (if successful) and new transcription-state + * 3b. On network error + * If retry-count < max then increment retry-count and retry the request + * Otherwise update the transcription-state in the database to 'transcription-failed' + * 4. Notify the callback that the work item is complete + * </pre> + */ +public class TranscriptionTask implements Runnable { + private static final String TAG = "TranscriptionTask"; + + private final Context context; + private final JobCallback callback; + private final JobWorkItem workItem; + private final TranscriptionClientFactory clientFactory; + private final Uri voicemailUri; + private final TranscriptionDbHelper databaseHelper; + private ByteString audioData; + private AudioFormat encoding; + + private static final int MAX_RETRIES = 2; + static final String AMR_PREFIX = "#!AMR\n"; + + public TranscriptionTask( + Context context, + JobCallback callback, + JobWorkItem workItem, + TranscriptionClientFactory clientFactory) { + this.context = context; + this.callback = callback; + this.workItem = workItem; + this.clientFactory = clientFactory; + this.voicemailUri = getVoicemailUri(workItem); + databaseHelper = new TranscriptionDbHelper(context, voicemailUri); + } + + @Override + public void run() { + VvmLog.i(TAG, "run"); + if (readAndValidateAudioFile()) { + updateTranscriptionState(VoicemailCompat.TRANSCRIPTION_IN_PROGRESS); + transcribeVoicemail(); + } else { + updateTranscriptionState(VoicemailCompat.TRANSCRIPTION_FAILED); + } + ThreadUtil.postOnUiThread( + () -> { + callback.onWorkCompleted(workItem); + }); + } + + private void transcribeVoicemail() { + VvmLog.i(TAG, "transcribeVoicemail"); + TranscribeVoicemailRequest request = makeRequest(); + TranscriptionClient client = clientFactory.getClient(); + String transcript = null; + for (int i = 0; transcript == null && i < MAX_RETRIES; i++) { + VvmLog.i(TAG, "transcribeVoicemail, try: " + (i + 1)); + TranscriptionClient.TranscriptionResponseWrapper responseWrapper = + client.transcribeVoicemail(request); + if (responseWrapper.status != null) { + VvmLog.i(TAG, "transcribeVoicemail, status: " + responseWrapper.status.getCode()); + if (shouldRetryRequest(responseWrapper.status)) { + backoff(i); + } else { + break; + } + } else if (responseWrapper.response != null) { + if (!TextUtils.isEmpty(responseWrapper.response.getTranscript())) { + VvmLog.i(TAG, "transcribeVoicemail, got response"); + transcript = responseWrapper.response.getTranscript(); + } else { + VvmLog.i(TAG, "transcribeVoicemail, empty transcription"); + } + } else { + VvmLog.w(TAG, "transcribeVoicemail, no response"); + } + } + + int newState = + (transcript == null) + ? VoicemailCompat.TRANSCRIPTION_FAILED + : VoicemailCompat.TRANSCRIPTION_AVAILABLE; + updateTranscriptionAndState(transcript, newState); + } + + private static boolean shouldRetryRequest(Status status) { + return status.getCode() == Status.Code.UNAVAILABLE; + } + + private static void backoff(int retryCount) { + VvmLog.i(TAG, "backoff, count: " + retryCount); + try { + long millis = (1 << retryCount) * 1000; + Thread.sleep(millis); + } catch (InterruptedException e) { + VvmLog.w(TAG, "interrupted"); + Thread.currentThread().interrupt(); + } + } + + private void updateTranscriptionAndState(String transcript, int newState) { + databaseHelper.setTranscriptionAndState(transcript, newState); + } + + private void updateTranscriptionState(int newState) { + databaseHelper.setTranscriptionState(newState); + } + + private TranscribeVoicemailRequest makeRequest() { + return TranscribeVoicemailRequest.newBuilder() + .setVoicemailData(audioData) + .setAudioFormat(encoding) + .build(); + } + + // Uses try-with-resource + @TargetApi(android.os.Build.VERSION_CODES.M) + private boolean readAndValidateAudioFile() { + if (voicemailUri == null) { + VvmLog.i(TAG, "Transcriber.readAndValidateAudioFile, file not found."); + return false; + } else { + VvmLog.i(TAG, "Transcriber.readAndValidateAudioFile, reading: " + voicemailUri); + } + + try (InputStream in = context.getContentResolver().openInputStream(voicemailUri)) { + audioData = ByteString.readFrom(in); + VvmLog.i(TAG, "Transcriber.readAndValidateAudioFile, read " + audioData.size() + " bytes"); + } catch (IOException e) { + VvmLog.e(TAG, "Transcriber.readAndValidateAudioFile", e); + return false; + } + + if (audioData.startsWith(ByteString.copyFromUtf8(AMR_PREFIX))) { + encoding = AudioFormat.AMR_NB_8KHZ; + } else { + VvmLog.i(TAG, "Transcriber.readAndValidateAudioFile, unknown encoding"); + encoding = AudioFormat.AUDIO_FORMAT_UNSPECIFIED; + return false; + } + + return true; + } + + private static Uri getVoicemailUri(JobWorkItem workItem) { + return workItem.getIntent().getParcelableExtra(TranscriptionService.EXTRA_VOICEMAIL_URI); + } +} diff --git a/java/com/android/voicemail/impl/transcribe/VoicemailCompat.java b/java/com/android/voicemail/impl/transcribe/VoicemailCompat.java new file mode 100644 index 000000000..c6e30c6de --- /dev/null +++ b/java/com/android/voicemail/impl/transcribe/VoicemailCompat.java @@ -0,0 +1,59 @@ +/* + * Copyright (C) 2017 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License + */ +package com.android.voicemail.impl.transcribe; + +/** + * Provide access to new API constants before they're publicly available + * + * <p>Copied from android.provider.VoicemailContract.Voicemails. These should become public in O-MR1 + * and these constants can be removed then. + */ +public class VoicemailCompat { + + /** + * The state of the voicemail transcription. + * + * <p>Possible values: {@link #TRANSCRIPTION_NOT_STARTED}, {@link #TRANSCRIPTION_IN_PROGRESS}, + * {@link #TRANSCRIPTION_FAILED}, {@link #TRANSCRIPTION_AVAILABLE}. + * + * <p>Type: INTEGER + */ + public static final String TRANSCRIPTION_STATE = "transcription_state"; + + /** + * Value of {@link #TRANSCRIPTION_STATE} when the voicemail transcription has not yet been + * attempted. + */ + public static final int TRANSCRIPTION_NOT_STARTED = 0; + + /** + * Value of {@link #TRANSCRIPTION_STATE} when the voicemail transcription has begun but is not yet + * complete. + */ + public static final int TRANSCRIPTION_IN_PROGRESS = 1; + + /** + * Value of {@link #TRANSCRIPTION_STATE} when the voicemail transcription has been attempted and + * failed. + */ + public static final int TRANSCRIPTION_FAILED = 2; + + /** + * Value of {@link #TRANSCRIPTION_STATE} when the voicemail transcription has completed and the + * result has been stored in the {@link #TRANSCRIPTION} column. + */ + public static final int TRANSCRIPTION_AVAILABLE = 3; +} diff --git a/java/com/android/voicemail/impl/transcribe/grpc/TranscriptionClient.java b/java/com/android/voicemail/impl/transcribe/grpc/TranscriptionClient.java new file mode 100644 index 000000000..27603d910 --- /dev/null +++ b/java/com/android/voicemail/impl/transcribe/grpc/TranscriptionClient.java @@ -0,0 +1,61 @@ +/* + * Copyright (C) 2017 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License + */ +package com.android.voicemail.impl.transcribe.grpc; + +import android.support.annotation.Nullable; +import android.support.annotation.WorkerThread; +import com.android.dialer.common.Assert; +import com.google.internal.communications.voicemailtranscription.v1.TranscribeVoicemailRequest; +import com.google.internal.communications.voicemailtranscription.v1.TranscribeVoicemailResponse; +import com.google.internal.communications.voicemailtranscription.v1.VoicemailTranscriptionServiceGrpc; +import io.grpc.Status; +import io.grpc.StatusRuntimeException; + +/** Wrapper around Grpc transcription server stub */ +public class TranscriptionClient { + + private final VoicemailTranscriptionServiceGrpc.VoicemailTranscriptionServiceBlockingStub stub; + + /** Wraps the server response and status objects, either of which may be null. */ + public static class TranscriptionResponseWrapper { + public final TranscribeVoicemailResponse response; + public final Status status; + + public TranscriptionResponseWrapper( + @Nullable TranscribeVoicemailResponse response, @Nullable Status status) { + Assert.checkArgument(!(response == null && status == null)); + this.response = response; + this.status = status; + } + } + + TranscriptionClient( + VoicemailTranscriptionServiceGrpc.VoicemailTranscriptionServiceBlockingStub stub) { + this.stub = stub; + } + + @WorkerThread + public TranscriptionResponseWrapper transcribeVoicemail(TranscribeVoicemailRequest request) { + TranscribeVoicemailResponse response = null; + Status status = null; + try { + response = stub.transcribeVoicemail(request); + } catch (StatusRuntimeException e) { + status = e.getStatus(); + } + return new TranscriptionClient.TranscriptionResponseWrapper(response, status); + } +} diff --git a/java/com/android/voicemail/impl/transcribe/grpc/TranscriptionClientFactory.java b/java/com/android/voicemail/impl/transcribe/grpc/TranscriptionClientFactory.java new file mode 100644 index 000000000..6101ed516 --- /dev/null +++ b/java/com/android/voicemail/impl/transcribe/grpc/TranscriptionClientFactory.java @@ -0,0 +1,194 @@ +/* + * Copyright (C) 2017 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License + */ +package com.android.voicemail.impl.transcribe.grpc; + +import android.content.Context; +import android.content.pm.PackageInfo; +import android.content.pm.PackageManager; +import android.text.TextUtils; +import com.android.dialer.common.Assert; +import com.android.dialer.common.LogUtil; +import com.android.voicemail.impl.transcribe.TranscriptionConfigProvider; +import com.google.internal.communications.voicemailtranscription.v1.VoicemailTranscriptionServiceGrpc; +import io.grpc.CallOptions; +import io.grpc.Channel; +import io.grpc.ClientCall; +import io.grpc.ClientInterceptor; +import io.grpc.ClientInterceptors; +import io.grpc.ForwardingClientCall; +import io.grpc.ManagedChannel; +import io.grpc.ManagedChannelBuilder; +import io.grpc.Metadata; +import io.grpc.MethodDescriptor; +import io.grpc.okhttp.OkHttpChannelBuilder; +import java.security.MessageDigest; + +/** + * Factory for creating grpc clients that talk to the transcription server. This allows all clients + * to share the same channel, which is relatively expensive to create. + */ +public class TranscriptionClientFactory { + private static final String DIGEST_ALGORITHM_SHA1 = "SHA1"; + private static final char[] HEX_UPPERCASE = { + '0', '1', '2', '3', '4', '5', '6', '7', '8', '9', 'A', 'B', 'C', 'D', 'E', 'F' + }; + + private final TranscriptionConfigProvider configProvider; + private final ManagedChannel originalChannel; + private final String packageName; + private final String cert; + + public TranscriptionClientFactory(Context context, TranscriptionConfigProvider configProvider) { + this(context, configProvider, getManagedChannel(configProvider)); + } + + public TranscriptionClientFactory( + Context context, TranscriptionConfigProvider configProvider, ManagedChannel managedChannel) { + this.configProvider = configProvider; + this.packageName = context.getPackageName(); + this.cert = getCertificateFingerprint(context); + originalChannel = managedChannel; + } + + public TranscriptionClient getClient() { + LogUtil.enterBlock("TranscriptionClientFactory.getClient"); + Assert.checkState(!originalChannel.isShutdown()); + Channel channel = + ClientInterceptors.intercept( + originalChannel, + new Interceptor( + packageName, cert, configProvider.getApiKey(), configProvider.getAuthToken())); + return new TranscriptionClient(VoicemailTranscriptionServiceGrpc.newBlockingStub(channel)); + } + + public void shutdown() { + LogUtil.enterBlock("TranscriptionClientFactory.shutdown"); + originalChannel.shutdown(); + } + + private static ManagedChannel getManagedChannel(TranscriptionConfigProvider configProvider) { + ManagedChannelBuilder<OkHttpChannelBuilder> builder = + OkHttpChannelBuilder.forTarget(configProvider.getServerAddress()); + // Only use plaintext for debugging + if (configProvider.shouldUsePlaintext()) { + // Just passing 'false' doesnt have the same effect as not setting this field + builder.usePlaintext(true); + } + return builder.build(); + } + + private static String getCertificateFingerprint(Context context) { + try { + PackageInfo packageInfo = + context + .getPackageManager() + .getPackageInfo(context.getPackageName(), PackageManager.GET_SIGNATURES); + if (packageInfo != null + && packageInfo.signatures != null + && packageInfo.signatures.length > 0) { + MessageDigest messageDigest = MessageDigest.getInstance(DIGEST_ALGORITHM_SHA1); + if (messageDigest == null) { + LogUtil.w( + "TranscriptionClientFactory.getCertificateFingerprint", "error getting digest."); + return null; + } + byte[] bytes = messageDigest.digest(packageInfo.signatures[0].toByteArray()); + if (bytes == null) { + LogUtil.w( + "TranscriptionClientFactory.getCertificateFingerprint", "empty message digest."); + return null; + } + + int length = bytes.length; + StringBuilder out = new StringBuilder(length * 2); + for (int i = 0; i < length; i++) { + out.append(HEX_UPPERCASE[(bytes[i] & 0xf0) >>> 4]); + out.append(HEX_UPPERCASE[bytes[i] & 0x0f]); + } + return out.toString(); + } else { + LogUtil.w( + "TranscriptionClientFactory.getCertificateFingerprint", + "failed to get package signature."); + } + } catch (Exception e) { + LogUtil.e( + "TranscriptionClientFactory.getCertificateFingerprint", + "error getting certificate fingerprint.", + e); + } + + return null; + } + + private static final class Interceptor implements ClientInterceptor { + private final String packageName; + private final String cert; + private final String apiKey; + private final String authToken; + + private static final Metadata.Key<String> API_KEY_HEADER = + Metadata.Key.of("X-Goog-Api-Key", Metadata.ASCII_STRING_MARSHALLER); + private static final Metadata.Key<String> ANDROID_PACKAGE_HEADER = + Metadata.Key.of("X-Android-Package", Metadata.ASCII_STRING_MARSHALLER); + private static final Metadata.Key<String> ANDROID_CERT_HEADER = + Metadata.Key.of("X-Android-Cert", Metadata.ASCII_STRING_MARSHALLER); + private static final Metadata.Key<String> AUTHORIZATION_HEADER = + Metadata.Key.of("authorization", Metadata.ASCII_STRING_MARSHALLER); + + public Interceptor(String packageName, String cert, String apiKey, String authToken) { + this.packageName = packageName; + this.cert = cert; + this.apiKey = apiKey; + this.authToken = authToken; + } + + @Override + public <ReqT, RespT> ClientCall<ReqT, RespT> interceptCall( + MethodDescriptor<ReqT, RespT> method, CallOptions callOptions, Channel next) { + LogUtil.enterBlock( + "TranscriptionClientFactory.interceptCall, intercepted " + method.getFullMethodName()); + ClientCall<ReqT, RespT> call = next.newCall(method, callOptions); + + call = + new ForwardingClientCall.SimpleForwardingClientCall<ReqT, RespT>(call) { + @Override + public void start(Listener<RespT> responseListener, Metadata headers) { + if (!TextUtils.isEmpty(packageName)) { + LogUtil.i( + "TranscriptionClientFactory.interceptCall", + "attaching package name: " + packageName); + headers.put(ANDROID_PACKAGE_HEADER, packageName); + } + if (!TextUtils.isEmpty(cert)) { + LogUtil.i("TranscriptionClientFactory.interceptCall", "attaching android cert"); + headers.put(ANDROID_CERT_HEADER, cert); + } + if (!TextUtils.isEmpty(apiKey)) { + LogUtil.i("TranscriptionClientFactory.interceptCall", "attaching API Key"); + headers.put(API_KEY_HEADER, apiKey); + } + if (!TextUtils.isEmpty(authToken)) { + LogUtil.i("TranscriptionClientFactory.interceptCall", "attaching auth token"); + headers.put(AUTHORIZATION_HEADER, "Bearer " + authToken); + } + super.start(responseListener, headers); + } + }; + return call; + } + } +} diff --git a/java/com/android/voicemail/impl/transcribe/grpc/voicemail_transcription.proto b/java/com/android/voicemail/impl/transcribe/grpc/voicemail_transcription.proto new file mode 100644 index 000000000..4b1e19b8a --- /dev/null +++ b/java/com/android/voicemail/impl/transcribe/grpc/voicemail_transcription.proto @@ -0,0 +1,44 @@ +// LINT.IfChange + +syntax = "proto2"; + +package google.internal.communications.voicemailtranscription.v1; + +option java_multiple_files = true; +option java_package = "com.google.internal.communications.voicemailtranscription.v1"; +option optimize_for = LITE_RUNTIME; + +// Enum that specifies supported audio formats. +enum AudioFormat { + // Default but invalid value. + AUDIO_FORMAT_UNSPECIFIED = 0; + + // Adaptive Multi-Rate Narrowband, 8kHz sampling frequency. + // https://en.wikipedia.org/wiki/Adaptive_Multi-Rate_audio_codec + AMR_NB_8KHZ = 1; +} + +// Request for synchronous voicemail transcription. +message TranscribeVoicemailRequest { + // Voicemail audio file containing the raw bytes we receive from the carrier. + optional bytes voicemail_data = 1; + + // Audio format of the voicemail file. + optional AudioFormat audio_format = 2; +} + +// Response for synchronous voicemail transcription. +message TranscribeVoicemailResponse { + // The transcribed text of the voicemail. + optional string transcript = 1; +} + +// RPC service for transcribing voicemails. +service VoicemailTranscriptionService { + // Returns a transcript of the given voicemail. + rpc TranscribeVoicemail(TranscribeVoicemailRequest) + returns (TranscribeVoicemailResponse) {} +} + +// LINT.ThenChange(//depot/google3/google/internal/communications/voicemailtranscription/v1/\ +// voicemail_transcription.proto) |