diff options
Diffstat (limited to 'java/com/android/voicemail/impl/transcribe')
15 files changed, 1652 insertions, 0 deletions
diff --git a/java/com/android/voicemail/impl/transcribe/TranscriptionBackfillService.java b/java/com/android/voicemail/impl/transcribe/TranscriptionBackfillService.java new file mode 100644 index 000000000..f3c6e64f4 --- /dev/null +++ b/java/com/android/voicemail/impl/transcribe/TranscriptionBackfillService.java @@ -0,0 +1,85 @@ +/* + * Copyright (C) 2017 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License + */ +package com.android.voicemail.impl.transcribe; + +import android.app.job.JobInfo; +import android.app.job.JobScheduler; +import android.app.job.JobWorkItem; +import android.content.ComponentName; +import android.content.Context; +import android.content.Intent; +import android.net.Uri; +import android.support.annotation.WorkerThread; +import android.support.v4.app.JobIntentService; +import android.support.v4.os.BuildCompat; +import com.android.dialer.common.LogUtil; +import com.android.dialer.common.concurrent.ThreadUtil; +import com.android.dialer.constants.ScheduledJobIds; +import java.util.List; + +/** + * JobScheduler service for transcribing old voicemails. This service does a database scan for + * un-transcribed voicemails and schedules transcription tasks for them, once we have an un-metered + * network connection. + */ +public class TranscriptionBackfillService extends JobIntentService { + + /** Schedule a task to scan the database for untranscribed voicemails */ + public static boolean scheduleTask(Context context) { + if (BuildCompat.isAtLeastO()) { + LogUtil.enterBlock("TranscriptionBackfillService.transcribeOldVoicemails"); + ComponentName componentName = new ComponentName(context, TranscriptionBackfillService.class); + JobInfo.Builder builder = + new JobInfo.Builder(ScheduledJobIds.VVM_TRANSCRIPTION_BACKFILL_JOB, componentName) + .setRequiredNetworkType(JobInfo.NETWORK_TYPE_UNMETERED); + JobScheduler scheduler = context.getSystemService(JobScheduler.class); + return scheduler.enqueue(builder.build(), makeWorkItem()) == JobScheduler.RESULT_SUCCESS; + } else { + LogUtil.i("TranscriptionBackfillService.transcribeOldVoicemails", "not supported"); + return false; + } + } + + private static JobWorkItem makeWorkItem() { + Intent intent = new Intent(); + return new JobWorkItem(intent); + } + + @Override + @WorkerThread + protected void onHandleWork(Intent intent) { + LogUtil.enterBlock("TranscriptionBackfillService.onHandleWork"); + + TranscriptionDbHelper dbHelper = new TranscriptionDbHelper(this); + List<Uri> untranscribed = dbHelper.getUntranscribedVoicemails(); + LogUtil.i( + "TranscriptionBackfillService.onHandleWork", + "found " + untranscribed.size() + " untranscribed voicemails"); + // TODO(mdooley): Consider doing the actual transcriptions here instead of scheduling jobs. + for (Uri uri : untranscribed) { + ThreadUtil.postOnUiThread( + () -> { + TranscriptionService.scheduleNewVoicemailTranscriptionJob(this, uri, false); + }); + } + } + + @Override + public void onDestroy() { + LogUtil.enterBlock("TranscriptionBackfillService.onDestroy"); + super.onDestroy(); + } +} diff --git a/java/com/android/voicemail/impl/transcribe/TranscriptionConfigProvider.java b/java/com/android/voicemail/impl/transcribe/TranscriptionConfigProvider.java new file mode 100644 index 000000000..83f04dab8 --- /dev/null +++ b/java/com/android/voicemail/impl/transcribe/TranscriptionConfigProvider.java @@ -0,0 +1,90 @@ +/* + * Copyright (C) 2017 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License + */ +package com.android.voicemail.impl.transcribe; + +import android.content.Context; +import com.android.dialer.configprovider.ConfigProviderBindings; + +/** Provides configuration values needed to connect to the transcription server. */ +public class TranscriptionConfigProvider { + private final Context context; + + public TranscriptionConfigProvider(Context context) { + this.context = context; + } + + public boolean isVoicemailTranscriptionEnabled() { + return ConfigProviderBindings.get(context).getBoolean("voicemail_transcription_enabled", false); + } + + public String getServerAddress() { + // Private voicemail transcription service + return ConfigProviderBindings.get(context) + .getString( + "voicemail_transcription_server_address", "voicemailtranscription-pa.googleapis.com"); + } + + public String getApiKey() { + // Android API key restricted to com.google.android.dialer + return ConfigProviderBindings.get(context) + .getString( + "voicemail_transcription_client_api_key", "AIzaSyAXdDnif6B7sBYxU8hzw9qAp3pRPVHs060"); + } + + public String getAuthToken() { + return null; + } + + public boolean shouldUsePlaintext() { + return ConfigProviderBindings.get(context) + .getBoolean("voicemail_transcription_server_use_plaintext", false); + } + + public boolean shouldUseSyncApi() { + return ConfigProviderBindings.get(context) + .getBoolean("voicemail_transcription_server_use_sync_api", false); + } + + public long getMaxTranscriptionRetries() { + return ConfigProviderBindings.get(context) + .getLong("voicemail_transcription_max_transcription_retries", 2L); + } + + public long getMaxGetTranscriptPolls() { + return ConfigProviderBindings.get(context) + .getLong("voicemail_transcription_max_get_transcript_polls", 20L); + } + + public long getGetTranscriptPollIntervalMillis() { + return ConfigProviderBindings.get(context) + .getLong("voicemail_transcription_get_transcript_poll_interval_millis", 1000L); + } + + @Override + public String toString() { + return String.format( + "{ address: %s, api key: %s, auth token: %s, plaintext: %b, sync: %b, retries: %d, polls:" + + " %d, poll ms: %d }", + getServerAddress(), + getApiKey(), + getAuthToken(), + shouldUsePlaintext(), + shouldUseSyncApi(), + getMaxTranscriptionRetries(), + getMaxGetTranscriptPolls(), + getGetTranscriptPollIntervalMillis()); + } +} diff --git a/java/com/android/voicemail/impl/transcribe/TranscriptionDbHelper.java b/java/com/android/voicemail/impl/transcribe/TranscriptionDbHelper.java new file mode 100644 index 000000000..9d3c2e4a7 --- /dev/null +++ b/java/com/android/voicemail/impl/transcribe/TranscriptionDbHelper.java @@ -0,0 +1,137 @@ +/* + * Copyright (C) 2017 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License + */ +package com.android.voicemail.impl.transcribe; + +import android.annotation.TargetApi; +import android.content.ContentResolver; +import android.content.ContentUris; +import android.content.ContentValues; +import android.content.Context; +import android.database.Cursor; +import android.net.Uri; +import android.os.Build.VERSION_CODES; +import android.provider.VoicemailContract.Voicemails; +import android.support.annotation.VisibleForTesting; +import android.support.annotation.WorkerThread; +import android.support.v4.os.BuildCompat; +import android.util.Pair; +import com.android.dialer.common.Assert; +import com.android.dialer.common.LogUtil; +import java.util.ArrayList; +import java.util.List; + +/** Helper class for reading and writing transcription data in the database */ +@TargetApi(VERSION_CODES.O) +public class TranscriptionDbHelper { + @VisibleForTesting + static final String[] PROJECTION = + new String[] { + Voicemails._ID, // 0 + Voicemails.TRANSCRIPTION, // 1 + VoicemailCompat.TRANSCRIPTION_STATE // 2 + }; + + static final int ID = 0; + static final int TRANSCRIPTION = 1; + static final int TRANSCRIPTION_STATE = 2; + + private final ContentResolver contentResolver; + private final Uri uri; + + TranscriptionDbHelper(Context context, Uri uri) { + Assert.isNotNull(uri); + this.contentResolver = context.getContentResolver(); + this.uri = uri; + } + + TranscriptionDbHelper(Context context) { + this(context, Voicemails.buildSourceUri(context.getPackageName())); + } + + @WorkerThread + @TargetApi(VERSION_CODES.M) // used for try with resources + Pair<String, Integer> getTranscriptionAndState() { + Assert.checkState(BuildCompat.isAtLeastO()); + Assert.isWorkerThread(); + try (Cursor cursor = contentResolver.query(uri, PROJECTION, null, null, null)) { + if (cursor == null) { + LogUtil.e("TranscriptionDbHelper.getTranscriptionAndState", "query failed."); + return null; + } + + if (cursor.moveToFirst()) { + String transcription = cursor.getString(TRANSCRIPTION); + int transcriptionState = cursor.getInt(TRANSCRIPTION_STATE); + return new Pair<>(transcription, transcriptionState); + } + } + LogUtil.i("TranscriptionDbHelper.getTranscriptionAndState", "query returned no results"); + return null; + } + + @WorkerThread + @TargetApi(VERSION_CODES.M) // used for try with resources + List<Uri> getUntranscribedVoicemails() { + Assert.checkArgument(BuildCompat.isAtLeastO()); + Assert.isWorkerThread(); + List<Uri> untranscribed = new ArrayList<>(); + String whereClause = + Voicemails.TRANSCRIPTION + " is NULL AND " + VoicemailCompat.TRANSCRIPTION_STATE + "=?"; + String[] whereArgs = {String.valueOf(VoicemailCompat.TRANSCRIPTION_NOT_STARTED)}; + try (Cursor cursor = contentResolver.query(uri, PROJECTION, whereClause, whereArgs, null)) { + if (cursor == null) { + LogUtil.e("TranscriptionDbHelper.getUntranscribedVoicemails", "query failed."); + } else { + while (cursor.moveToNext()) { + untranscribed.add(ContentUris.withAppendedId(uri, cursor.getLong(ID))); + } + } + } + return untranscribed; + } + + @WorkerThread + void setTranscriptionState(int transcriptionState) { + Assert.isWorkerThread(); + LogUtil.i( + "TranscriptionDbHelper.setTranscriptionState", + "uri: " + uri + ", state: " + transcriptionState); + ContentValues values = new ContentValues(); + values.put(VoicemailCompat.TRANSCRIPTION_STATE, transcriptionState); + updateDatabase(values); + } + + @WorkerThread + void setTranscriptionAndState(String transcription, int transcriptionState) { + Assert.isWorkerThread(); + LogUtil.i( + "TranscriptionDbHelper.setTranscriptionAndState", + "uri: " + uri + ", state: " + transcriptionState); + ContentValues values = new ContentValues(); + values.put(Voicemails.TRANSCRIPTION, transcription); + values.put(VoicemailCompat.TRANSCRIPTION_STATE, transcriptionState); + updateDatabase(values); + } + + private void updateDatabase(ContentValues values) { + int updatedCount = contentResolver.update(uri, values, null, null); + if (updatedCount != 1) { + LogUtil.e( + "TranscriptionDbHelper.updateDatabase", + "Wrong row count, should have updated 1 row, was: " + updatedCount); + } + } +} diff --git a/java/com/android/voicemail/impl/transcribe/TranscriptionService.java b/java/com/android/voicemail/impl/transcribe/TranscriptionService.java new file mode 100644 index 000000000..2ca16fbf2 --- /dev/null +++ b/java/com/android/voicemail/impl/transcribe/TranscriptionService.java @@ -0,0 +1,223 @@ +/* + * Copyright (C) 2017 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License + */ +package com.android.voicemail.impl.transcribe; + +import android.app.job.JobInfo; +import android.app.job.JobParameters; +import android.app.job.JobScheduler; +import android.app.job.JobService; +import android.app.job.JobWorkItem; +import android.content.ComponentName; +import android.content.Context; +import android.content.Intent; +import android.net.Uri; +import android.support.annotation.MainThread; +import android.support.annotation.VisibleForTesting; +import android.support.v4.os.BuildCompat; +import android.text.TextUtils; +import com.android.dialer.common.Assert; +import com.android.dialer.common.LogUtil; +import com.android.dialer.constants.ScheduledJobIds; +import com.android.dialer.logging.DialerImpression; +import com.android.dialer.logging.Logger; +import com.android.voicemail.impl.transcribe.grpc.TranscriptionClientFactory; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; + +/** + * Job scheduler callback for launching voicemail transcription tasks. The transcription tasks will + * run in the background and will typically last for approximately the length of the voicemail audio + * (since thats how long the backend transcription service takes to do the transcription). + */ +public class TranscriptionService extends JobService { + @VisibleForTesting static final String EXTRA_VOICEMAIL_URI = "extra_voicemail_uri"; + + private ExecutorService executorService; + private JobParameters jobParameters; + private TranscriptionClientFactory clientFactory; + private TranscriptionConfigProvider configProvider; + + /** Callback used by a task to indicate it has finished processing its work item */ + interface JobCallback { + void onWorkCompleted(JobWorkItem completedWorkItem); + } + + // Schedule a task to transcribe the indicated voicemail, return true if transcription task was + // scheduled. + @MainThread + public static boolean scheduleNewVoicemailTranscriptionJob( + Context context, Uri voicemailUri, boolean highPriority) { + Assert.isMainThread(); + if (BuildCompat.isAtLeastO()) { + LogUtil.i( + "TranscriptionService.scheduleNewVoicemailTranscriptionJob", "scheduling transcription"); + Logger.get(context).logImpression(DialerImpression.Type.VVM_TRANSCRIPTION_VOICEMAIL_RECEIVED); + + ComponentName componentName = new ComponentName(context, TranscriptionService.class); + JobInfo.Builder builder = + new JobInfo.Builder(ScheduledJobIds.VVM_TRANSCRIPTION_JOB, componentName); + if (highPriority) { + builder + .setMinimumLatency(0) + .setOverrideDeadline(0) + .setRequiredNetworkType(JobInfo.NETWORK_TYPE_ANY); + } else { + builder.setRequiredNetworkType(JobInfo.NETWORK_TYPE_UNMETERED); + } + JobScheduler scheduler = context.getSystemService(JobScheduler.class); + JobWorkItem workItem = makeWorkItem(voicemailUri); + return scheduler.enqueue(builder.build(), workItem) == JobScheduler.RESULT_SUCCESS; + } else { + LogUtil.i("TranscriptionService.scheduleNewVoicemailTranscriptionJob", "not supported"); + return false; + } + } + + // Cancel all transcription tasks + @MainThread + public static void cancelTranscriptions(Context context) { + Assert.isMainThread(); + LogUtil.enterBlock("TranscriptionService.cancelTranscriptions"); + JobScheduler scheduler = context.getSystemService(JobScheduler.class); + scheduler.cancel(ScheduledJobIds.VVM_TRANSCRIPTION_JOB); + } + + @MainThread + public TranscriptionService() { + Assert.isMainThread(); + } + + @VisibleForTesting + TranscriptionService( + ExecutorService executorService, + TranscriptionClientFactory clientFactory, + TranscriptionConfigProvider configProvider) { + this.executorService = executorService; + this.clientFactory = clientFactory; + this.configProvider = configProvider; + } + + @Override + @MainThread + public boolean onStartJob(JobParameters params) { + Assert.isMainThread(); + LogUtil.enterBlock("TranscriptionService.onStartJob"); + if (!getConfigProvider().isVoicemailTranscriptionEnabled()) { + LogUtil.i("TranscriptionService.onStartJob", "transcription not enabled, exiting."); + return false; + } else if (TextUtils.isEmpty(getConfigProvider().getServerAddress())) { + LogUtil.i("TranscriptionService.onStartJob", "transcription server not configured, exiting."); + return false; + } else { + LogUtil.i( + "TranscriptionService.onStartJob", + "transcription server address: " + configProvider.getServerAddress()); + jobParameters = params; + return checkForWork(); + } + } + + @Override + @MainThread + public boolean onStopJob(JobParameters params) { + Assert.isMainThread(); + LogUtil.enterBlock("TranscriptionService.onStopJob"); + cleanup(); + return true; + } + + @Override + @MainThread + public void onDestroy() { + Assert.isMainThread(); + LogUtil.enterBlock("TranscriptionService.onDestroy"); + cleanup(); + } + + private void cleanup() { + if (clientFactory != null) { + clientFactory.shutdown(); + clientFactory = null; + } + if (executorService != null) { + executorService.shutdownNow(); + executorService = null; + } + } + + @MainThread + private boolean checkForWork() { + Assert.isMainThread(); + JobWorkItem workItem = jobParameters.dequeueWork(); + if (workItem != null) { + TranscriptionTask task = + configProvider.shouldUseSyncApi() + ? new TranscriptionTaskSync( + this, new Callback(), workItem, getClientFactory(), configProvider) + : new TranscriptionTaskAsync( + this, new Callback(), workItem, getClientFactory(), configProvider); + getExecutorService().execute(task); + return true; + } else { + return false; + } + } + + static Uri getVoicemailUri(JobWorkItem workItem) { + return workItem.getIntent().getParcelableExtra(EXTRA_VOICEMAIL_URI); + } + + private ExecutorService getExecutorService() { + if (executorService == null) { + // The common use case is transcribing a single voicemail so just use a single thread executor + // The reason we're not using DialerExecutor here is because the transcription task can be + // very long running (ie. multiple minutes). + executorService = Executors.newSingleThreadExecutor(); + } + return executorService; + } + + private class Callback implements JobCallback { + @Override + @MainThread + public void onWorkCompleted(JobWorkItem completedWorkItem) { + Assert.isMainThread(); + LogUtil.i("TranscriptionService.Callback.onWorkCompleted", completedWorkItem.toString()); + jobParameters.completeWork(completedWorkItem); + checkForWork(); + } + } + + private static JobWorkItem makeWorkItem(Uri voicemailUri) { + Intent intent = new Intent(); + intent.putExtra(EXTRA_VOICEMAIL_URI, voicemailUri); + return new JobWorkItem(intent); + } + + private TranscriptionConfigProvider getConfigProvider() { + if (configProvider == null) { + configProvider = new TranscriptionConfigProvider(this); + } + return configProvider; + } + + private TranscriptionClientFactory getClientFactory() { + if (clientFactory == null) { + clientFactory = new TranscriptionClientFactory(this, getConfigProvider()); + } + return clientFactory; + } +} diff --git a/java/com/android/voicemail/impl/transcribe/TranscriptionTask.java b/java/com/android/voicemail/impl/transcribe/TranscriptionTask.java new file mode 100644 index 000000000..b5f29da00 --- /dev/null +++ b/java/com/android/voicemail/impl/transcribe/TranscriptionTask.java @@ -0,0 +1,225 @@ +/* + * Copyright (C) 2017 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License + */ +package com.android.voicemail.impl.transcribe; + +import android.annotation.TargetApi; +import android.app.job.JobWorkItem; +import android.content.Context; +import android.net.Uri; +import android.text.TextUtils; +import android.util.Pair; +import com.android.dialer.common.concurrent.ThreadUtil; +import com.android.dialer.logging.DialerImpression; +import com.android.dialer.logging.Logger; +import com.android.voicemail.impl.VvmLog; +import com.android.voicemail.impl.transcribe.TranscriptionService.JobCallback; +import com.android.voicemail.impl.transcribe.grpc.TranscriptionClient; +import com.android.voicemail.impl.transcribe.grpc.TranscriptionClientFactory; +import com.android.voicemail.impl.transcribe.grpc.TranscriptionResponse; +import com.google.internal.communications.voicemailtranscription.v1.AudioFormat; +import com.google.internal.communications.voicemailtranscription.v1.TranscriptionStatus; +import com.google.protobuf.ByteString; +import java.io.IOException; +import java.io.InputStream; + +/** + * Background task to get a voicemail transcription and update the database. + * + * <pre> + * This task performs the following steps: + * 1. Update the transcription-state in the database to 'in-progress' + * 2. Create grpc client and transcription request + * 3. Make synchronous or asynchronous grpc transcription request to backend server + * 3a. On response + * Update the database with transcription (if successful) and new transcription-state + * 3b. On network error + * If retry-count < max then increment retry-count and retry the request + * Otherwise update the transcription-state in the database to 'transcription-failed' + * 4. Notify the callback that the work item is complete + * </pre> + */ +public abstract class TranscriptionTask implements Runnable { + private static final String TAG = "TranscriptionTask"; + + private final Context context; + private final JobCallback callback; + private final JobWorkItem workItem; + private final TranscriptionClientFactory clientFactory; + private final Uri voicemailUri; + private final TranscriptionDbHelper databaseHelper; + protected final TranscriptionConfigProvider configProvider; + protected ByteString audioData; + protected AudioFormat encoding; + + static final String AMR_PREFIX = "#!AMR\n"; + + /** Functional interface for sending requests to the transcription server */ + public interface Request { + TranscriptionResponse getResponse(TranscriptionClient client); + } + + TranscriptionTask( + Context context, + JobCallback callback, + JobWorkItem workItem, + TranscriptionClientFactory clientFactory, + TranscriptionConfigProvider configProvider) { + this.context = context; + this.callback = callback; + this.workItem = workItem; + this.clientFactory = clientFactory; + this.voicemailUri = TranscriptionService.getVoicemailUri(workItem); + this.configProvider = configProvider; + databaseHelper = new TranscriptionDbHelper(context, voicemailUri); + } + + @Override + public void run() { + VvmLog.i(TAG, "run"); + if (readAndValidateAudioFile()) { + updateTranscriptionState(VoicemailCompat.TRANSCRIPTION_IN_PROGRESS); + transcribeVoicemail(); + } else { + if (AudioFormat.AUDIO_FORMAT_UNSPECIFIED.equals(encoding)) { + Logger.get(context) + .logImpression(DialerImpression.Type.VVM_TRANSCRIPTION_VOICEMAIL_FORMAT_NOT_SUPPORTED); + } else { + Logger.get(context) + .logImpression(DialerImpression.Type.VVM_TRANSCRIPTION_VOICEMAIL_INVALID_DATA); + } + updateTranscriptionState(VoicemailCompat.TRANSCRIPTION_FAILED); + } + ThreadUtil.postOnUiThread( + () -> { + callback.onWorkCompleted(workItem); + }); + } + + protected abstract Pair<String, TranscriptionStatus> getTranscription(); + + protected abstract DialerImpression.Type getRequestSentImpression(); + + private void transcribeVoicemail() { + VvmLog.i(TAG, "transcribeVoicemail"); + Pair<String, TranscriptionStatus> pair = getTranscription(); + String transcript = pair.first; + TranscriptionStatus status = pair.second; + if (!TextUtils.isEmpty(transcript)) { + updateTranscriptionAndState(transcript, VoicemailCompat.TRANSCRIPTION_AVAILABLE); + VvmLog.i(TAG, "transcribeVoicemail, got response"); + Logger.get(context).logImpression(DialerImpression.Type.VVM_TRANSCRIPTION_RESPONSE_SUCCESS); + } else { + VvmLog.i(TAG, "transcribeVoicemail, transcription unsuccessful, " + status); + switch (status) { + case FAILED_LANGUAGE_NOT_SUPPORTED: + Logger.get(context) + .logImpression( + DialerImpression.Type.VVM_TRANSCRIPTION_RESPONSE_LANGUAGE_NOT_SUPPORTED); + break; + case FAILED_NO_SPEECH_DETECTED: + Logger.get(context) + .logImpression(DialerImpression.Type.VVM_TRANSCRIPTION_RESPONSE_NO_SPEECH_DETECTED); + break; + case EXPIRED: + Logger.get(context) + .logImpression(DialerImpression.Type.VVM_TRANSCRIPTION_RESPONSE_EXPIRED); + break; + default: + Logger.get(context).logImpression(DialerImpression.Type.VVM_TRANSCRIPTION_RESPONSE_EMPTY); + break; + } + updateTranscriptionAndState(transcript, VoicemailCompat.TRANSCRIPTION_FAILED); + } + } + + protected TranscriptionResponse sendRequest(Request request) { + VvmLog.i(TAG, "sendRequest"); + TranscriptionClient client = clientFactory.getClient(); + for (int i = 0; i < configProvider.getMaxTranscriptionRetries(); i++) { + VvmLog.i(TAG, "sendRequest, try: " + (i + 1)); + if (i == 0) { + Logger.get(context).logImpression(getRequestSentImpression()); + } else { + Logger.get(context).logImpression(DialerImpression.Type.VVM_TRANSCRIPTION_REQUEST_RETRY); + } + + TranscriptionResponse response = request.getResponse(client); + if (response.hasRecoverableError()) { + Logger.get(context) + .logImpression(DialerImpression.Type.VVM_TRANSCRIPTION_RESPONSE_RECOVERABLE_ERROR); + backoff(i); + } else { + return response; + } + } + + Logger.get(context) + .logImpression(DialerImpression.Type.VVM_TRANSCRIPTION_RESPONSE_TOO_MANY_ERRORS); + return null; + } + + private static void backoff(int retryCount) { + VvmLog.i(TAG, "backoff, count: " + retryCount); + long millis = (1L << retryCount) * 1000; + sleep(millis); + } + + protected static void sleep(long millis) { + try { + Thread.sleep(millis); + } catch (InterruptedException e) { + VvmLog.w(TAG, "interrupted"); + Thread.currentThread().interrupt(); + } + } + + private void updateTranscriptionAndState(String transcript, int newState) { + databaseHelper.setTranscriptionAndState(transcript, newState); + } + + private void updateTranscriptionState(int newState) { + databaseHelper.setTranscriptionState(newState); + } + + // Uses try-with-resource + @TargetApi(android.os.Build.VERSION_CODES.M) + private boolean readAndValidateAudioFile() { + if (voicemailUri == null) { + VvmLog.i(TAG, "Transcriber.readAndValidateAudioFile, file not found."); + return false; + } else { + VvmLog.i(TAG, "Transcriber.readAndValidateAudioFile, reading: " + voicemailUri); + } + + try (InputStream in = context.getContentResolver().openInputStream(voicemailUri)) { + audioData = ByteString.readFrom(in); + VvmLog.i(TAG, "Transcriber.readAndValidateAudioFile, read " + audioData.size() + " bytes"); + } catch (IOException e) { + VvmLog.e(TAG, "Transcriber.readAndValidateAudioFile", e); + return false; + } + + if (audioData.startsWith(ByteString.copyFromUtf8(AMR_PREFIX))) { + encoding = AudioFormat.AMR_NB_8KHZ; + } else { + VvmLog.i(TAG, "Transcriber.readAndValidateAudioFile, unknown encoding"); + encoding = AudioFormat.AUDIO_FORMAT_UNSPECIFIED; + return false; + } + + return true; + } +} diff --git a/java/com/android/voicemail/impl/transcribe/TranscriptionTaskAsync.java b/java/com/android/voicemail/impl/transcribe/TranscriptionTaskAsync.java new file mode 100644 index 000000000..3c41aef89 --- /dev/null +++ b/java/com/android/voicemail/impl/transcribe/TranscriptionTaskAsync.java @@ -0,0 +1,123 @@ +/* + * Copyright (C) 2017 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License + */ +package com.android.voicemail.impl.transcribe; + +import android.app.job.JobWorkItem; +import android.content.Context; +import android.util.Pair; +import com.android.dialer.common.Assert; +import com.android.dialer.logging.DialerImpression; +import com.android.voicemail.impl.VvmLog; +import com.android.voicemail.impl.transcribe.TranscriptionService.JobCallback; +import com.android.voicemail.impl.transcribe.grpc.GetTranscriptResponseAsync; +import com.android.voicemail.impl.transcribe.grpc.TranscriptionClientFactory; +import com.android.voicemail.impl.transcribe.grpc.TranscriptionResponseAsync; +import com.google.internal.communications.voicemailtranscription.v1.GetTranscriptRequest; +import com.google.internal.communications.voicemailtranscription.v1.TranscribeVoicemailAsyncRequest; +import com.google.internal.communications.voicemailtranscription.v1.TranscriptionStatus; + +/** + * Background task to get a voicemail transcription using the asynchronous API. The async API works + * as follows: + * + * <ol> + * <li>client uploads voicemail data to the server + * <li>server responds with a transcription-id and an estimated transcription wait time + * <li>client waits appropriate amount of time then begins polling for the result + * </ol> + * + * This implementation blocks until the response or an error is received, even though it is using + * the asynchronous server API. + */ +public class TranscriptionTaskAsync extends TranscriptionTask { + private static final String TAG = "TranscriptionTaskAsync"; + + public TranscriptionTaskAsync( + Context context, + JobCallback callback, + JobWorkItem workItem, + TranscriptionClientFactory clientFactory, + TranscriptionConfigProvider configProvider) { + super(context, callback, workItem, clientFactory, configProvider); + } + + @Override + protected Pair<String, TranscriptionStatus> getTranscription() { + VvmLog.i(TAG, "getTranscription"); + + TranscriptionResponseAsync uploadResponse = + (TranscriptionResponseAsync) + sendRequest((client) -> client.sendUploadRequest(getUploadRequest())); + + if (uploadResponse == null) { + VvmLog.i(TAG, "getTranscription, failed to upload voicemail."); + return new Pair<>(null, TranscriptionStatus.FAILED_NO_RETRY); + } else { + waitForTranscription(uploadResponse); + return pollForTranscription(uploadResponse); + } + } + + @Override + protected DialerImpression.Type getRequestSentImpression() { + return DialerImpression.Type.VVM_TRANSCRIPTION_REQUEST_SENT_ASYNC; + } + + private static void waitForTranscription(TranscriptionResponseAsync uploadResponse) { + long millis = uploadResponse.getEstimatedWaitMillis(); + VvmLog.i(TAG, "waitForTranscription, " + millis + " millis"); + sleep(millis); + } + + private Pair<String, TranscriptionStatus> pollForTranscription( + TranscriptionResponseAsync uploadResponse) { + VvmLog.i(TAG, "pollForTranscription"); + GetTranscriptRequest request = getGetTranscriptRequest(uploadResponse); + for (int i = 0; i < configProvider.getMaxGetTranscriptPolls(); i++) { + GetTranscriptResponseAsync response = + (GetTranscriptResponseAsync) + sendRequest((client) -> client.sendGetTranscriptRequest(request)); + if (response == null) { + VvmLog.i(TAG, "pollForTranscription, no transcription result."); + } else if (response.isTranscribing()) { + VvmLog.i(TAG, "pollForTranscription, poll count: " + (i + 1)); + } else if (response.hasFatalError()) { + VvmLog.i(TAG, "pollForTranscription, fail. " + response.getErrorDescription()); + return new Pair<>(null, response.getTranscriptionStatus()); + } else { + VvmLog.i(TAG, "pollForTranscription, got transcription"); + return new Pair<>(response.getTranscript(), TranscriptionStatus.SUCCESS); + } + sleep(configProvider.getGetTranscriptPollIntervalMillis()); + } + VvmLog.i(TAG, "pollForTranscription, timed out."); + return new Pair<>(null, TranscriptionStatus.FAILED_NO_RETRY); + } + + private TranscribeVoicemailAsyncRequest getUploadRequest() { + return TranscribeVoicemailAsyncRequest.newBuilder() + .setVoicemailData(audioData) + .setAudioFormat(encoding) + .build(); + } + + private GetTranscriptRequest getGetTranscriptRequest(TranscriptionResponseAsync uploadResponse) { + Assert.checkArgument(uploadResponse.getTranscriptionId() != null); + return GetTranscriptRequest.newBuilder() + .setTranscriptionId(uploadResponse.getTranscriptionId()) + .build(); + } +} diff --git a/java/com/android/voicemail/impl/transcribe/TranscriptionTaskSync.java b/java/com/android/voicemail/impl/transcribe/TranscriptionTaskSync.java new file mode 100644 index 000000000..bee68590a --- /dev/null +++ b/java/com/android/voicemail/impl/transcribe/TranscriptionTaskSync.java @@ -0,0 +1,69 @@ +/* + * Copyright (C) 2017 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License + */ +package com.android.voicemail.impl.transcribe; + +import android.app.job.JobWorkItem; +import android.content.Context; +import android.util.Pair; +import com.android.dialer.logging.DialerImpression; +import com.android.voicemail.impl.VvmLog; +import com.android.voicemail.impl.transcribe.TranscriptionService.JobCallback; +import com.android.voicemail.impl.transcribe.grpc.TranscriptionClientFactory; +import com.android.voicemail.impl.transcribe.grpc.TranscriptionResponseSync; +import com.google.internal.communications.voicemailtranscription.v1.TranscribeVoicemailRequest; +import com.google.internal.communications.voicemailtranscription.v1.TranscriptionStatus; + +/** Background task to get a voicemail transcription using the synchronous API */ +public class TranscriptionTaskSync extends TranscriptionTask { + private static final String TAG = "TranscriptionTaskSync"; + + public TranscriptionTaskSync( + Context context, + JobCallback callback, + JobWorkItem workItem, + TranscriptionClientFactory clientFactory, + TranscriptionConfigProvider configProvider) { + super(context, callback, workItem, clientFactory, configProvider); + } + + @Override + protected Pair<String, TranscriptionStatus> getTranscription() { + VvmLog.i(TAG, "getTranscription"); + + TranscriptionResponseSync response = + (TranscriptionResponseSync) + sendRequest((client) -> client.sendSyncRequest(getSyncRequest())); + if (response == null) { + VvmLog.i(TAG, "getTranscription, failed to transcribe voicemail."); + return new Pair<>(null, TranscriptionStatus.FAILED_NO_RETRY); + } else { + VvmLog.i(TAG, "getTranscription, got transcription"); + return new Pair<>(response.getTranscript(), TranscriptionStatus.SUCCESS); + } + } + + @Override + protected DialerImpression.Type getRequestSentImpression() { + return DialerImpression.Type.VVM_TRANSCRIPTION_REQUEST_SENT; + } + + private TranscribeVoicemailRequest getSyncRequest() { + return TranscribeVoicemailRequest.newBuilder() + .setVoicemailData(audioData) + .setAudioFormat(encoding) + .build(); + } +} diff --git a/java/com/android/voicemail/impl/transcribe/VoicemailCompat.java b/java/com/android/voicemail/impl/transcribe/VoicemailCompat.java new file mode 100644 index 000000000..c6e30c6de --- /dev/null +++ b/java/com/android/voicemail/impl/transcribe/VoicemailCompat.java @@ -0,0 +1,59 @@ +/* + * Copyright (C) 2017 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License + */ +package com.android.voicemail.impl.transcribe; + +/** + * Provide access to new API constants before they're publicly available + * + * <p>Copied from android.provider.VoicemailContract.Voicemails. These should become public in O-MR1 + * and these constants can be removed then. + */ +public class VoicemailCompat { + + /** + * The state of the voicemail transcription. + * + * <p>Possible values: {@link #TRANSCRIPTION_NOT_STARTED}, {@link #TRANSCRIPTION_IN_PROGRESS}, + * {@link #TRANSCRIPTION_FAILED}, {@link #TRANSCRIPTION_AVAILABLE}. + * + * <p>Type: INTEGER + */ + public static final String TRANSCRIPTION_STATE = "transcription_state"; + + /** + * Value of {@link #TRANSCRIPTION_STATE} when the voicemail transcription has not yet been + * attempted. + */ + public static final int TRANSCRIPTION_NOT_STARTED = 0; + + /** + * Value of {@link #TRANSCRIPTION_STATE} when the voicemail transcription has begun but is not yet + * complete. + */ + public static final int TRANSCRIPTION_IN_PROGRESS = 1; + + /** + * Value of {@link #TRANSCRIPTION_STATE} when the voicemail transcription has been attempted and + * failed. + */ + public static final int TRANSCRIPTION_FAILED = 2; + + /** + * Value of {@link #TRANSCRIPTION_STATE} when the voicemail transcription has completed and the + * result has been stored in the {@link #TRANSCRIPTION} column. + */ + public static final int TRANSCRIPTION_AVAILABLE = 3; +} diff --git a/java/com/android/voicemail/impl/transcribe/grpc/GetTranscriptResponseAsync.java b/java/com/android/voicemail/impl/transcribe/grpc/GetTranscriptResponseAsync.java new file mode 100644 index 000000000..f979d69ce --- /dev/null +++ b/java/com/android/voicemail/impl/transcribe/grpc/GetTranscriptResponseAsync.java @@ -0,0 +1,102 @@ +/* + * Copyright (C) 2017 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License + */ +package com.android.voicemail.impl.transcribe.grpc; + +import android.support.annotation.Nullable; +import android.support.annotation.VisibleForTesting; +import com.android.dialer.common.Assert; +import com.google.internal.communications.voicemailtranscription.v1.GetTranscriptResponse; +import com.google.internal.communications.voicemailtranscription.v1.TranscriptionStatus; +import io.grpc.Status; + +/** Container for response and status objects for an asynchronous get-transcript request */ +public class GetTranscriptResponseAsync extends TranscriptionResponse { + @Nullable private final GetTranscriptResponse response; + + @VisibleForTesting + public GetTranscriptResponseAsync(GetTranscriptResponse response) { + Assert.checkArgument(response != null); + this.response = response; + } + + @VisibleForTesting + public GetTranscriptResponseAsync(Status status) { + super(status); + this.response = null; + } + + public @Nullable String getTranscript() { + if (response != null) { + return response.getTranscript(); + } + return null; + } + + public @Nullable String getErrorDescription() { + if (!hasRecoverableError() && !hasFatalError()) { + return null; + } + if (status != null) { + return "Grpc error: " + status; + } + if (response != null) { + return "Transcription error: " + response.getStatus(); + } + Assert.fail("Impossible state"); + return null; + } + + public TranscriptionStatus getTranscriptionStatus() { + if (response == null) { + return TranscriptionStatus.TRANSCRIPTION_STATUS_UNSPECIFIED; + } else { + return response.getStatus(); + } + } + + public boolean isTranscribing() { + return response != null && response.getStatus() == TranscriptionStatus.PENDING; + } + + @Override + public boolean hasRecoverableError() { + if (super.hasRecoverableError()) { + return true; + } + + if (response != null) { + return response.getStatus() == TranscriptionStatus.EXPIRED + || response.getStatus() == TranscriptionStatus.FAILED_RETRY; + } + + return false; + } + + @Override + public boolean hasFatalError() { + if (super.hasFatalError()) { + return true; + } + + if (response != null) { + return response.getStatus() == TranscriptionStatus.FAILED_NO_RETRY + || response.getStatus() == TranscriptionStatus.FAILED_LANGUAGE_NOT_SUPPORTED + || response.getStatus() == TranscriptionStatus.FAILED_NO_SPEECH_DETECTED; + } + + return false; + } +} diff --git a/java/com/android/voicemail/impl/transcribe/grpc/TranscriptionClient.java b/java/com/android/voicemail/impl/transcribe/grpc/TranscriptionClient.java new file mode 100644 index 000000000..b18d95627 --- /dev/null +++ b/java/com/android/voicemail/impl/transcribe/grpc/TranscriptionClient.java @@ -0,0 +1,61 @@ +/* + * Copyright (C) 2017 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License + */ +package com.android.voicemail.impl.transcribe.grpc; + +import android.support.annotation.WorkerThread; +import com.google.internal.communications.voicemailtranscription.v1.GetTranscriptRequest; +import com.google.internal.communications.voicemailtranscription.v1.TranscribeVoicemailAsyncRequest; +import com.google.internal.communications.voicemailtranscription.v1.TranscribeVoicemailRequest; +import com.google.internal.communications.voicemailtranscription.v1.VoicemailTranscriptionServiceGrpc; +import io.grpc.StatusRuntimeException; + +/** Wrapper around Grpc transcription server stub */ +public class TranscriptionClient { + + private final VoicemailTranscriptionServiceGrpc.VoicemailTranscriptionServiceBlockingStub stub; + + TranscriptionClient( + VoicemailTranscriptionServiceGrpc.VoicemailTranscriptionServiceBlockingStub stub) { + this.stub = stub; + } + + @WorkerThread + public TranscriptionResponseSync sendSyncRequest(TranscribeVoicemailRequest request) { + try { + return new TranscriptionResponseSync(stub.transcribeVoicemail(request)); + } catch (StatusRuntimeException e) { + return new TranscriptionResponseSync(e.getStatus()); + } + } + + @WorkerThread + public TranscriptionResponseAsync sendUploadRequest(TranscribeVoicemailAsyncRequest request) { + try { + return new TranscriptionResponseAsync(stub.transcribeVoicemailAsync(request)); + } catch (StatusRuntimeException e) { + return new TranscriptionResponseAsync(e.getStatus()); + } + } + + @WorkerThread + public GetTranscriptResponseAsync sendGetTranscriptRequest(GetTranscriptRequest request) { + try { + return new GetTranscriptResponseAsync(stub.getTranscript(request)); + } catch (StatusRuntimeException e) { + return new GetTranscriptResponseAsync(e.getStatus()); + } + } +} diff --git a/java/com/android/voicemail/impl/transcribe/grpc/TranscriptionClientFactory.java b/java/com/android/voicemail/impl/transcribe/grpc/TranscriptionClientFactory.java new file mode 100644 index 000000000..c57b01fd7 --- /dev/null +++ b/java/com/android/voicemail/impl/transcribe/grpc/TranscriptionClientFactory.java @@ -0,0 +1,196 @@ +/* + * Copyright (C) 2017 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License + */ +package com.android.voicemail.impl.transcribe.grpc; + +import android.content.Context; +import android.content.pm.PackageInfo; +import android.content.pm.PackageManager; +import android.text.TextUtils; +import com.android.dialer.common.Assert; +import com.android.dialer.common.LogUtil; +import com.android.voicemail.impl.transcribe.TranscriptionConfigProvider; +import com.google.internal.communications.voicemailtranscription.v1.VoicemailTranscriptionServiceGrpc; +import io.grpc.CallOptions; +import io.grpc.Channel; +import io.grpc.ClientCall; +import io.grpc.ClientInterceptor; +import io.grpc.ClientInterceptors; +import io.grpc.ForwardingClientCall; +import io.grpc.ManagedChannel; +import io.grpc.ManagedChannelBuilder; +import io.grpc.Metadata; +import io.grpc.MethodDescriptor; +import io.grpc.okhttp.OkHttpChannelBuilder; +import java.security.MessageDigest; + +/** + * Factory for creating grpc clients that talk to the transcription server. This allows all clients + * to share the same channel, which is relatively expensive to create. + */ +public class TranscriptionClientFactory { + private static final String DIGEST_ALGORITHM_SHA1 = "SHA1"; + private static final char[] HEX_UPPERCASE = { + '0', '1', '2', '3', '4', '5', '6', '7', '8', '9', 'A', 'B', 'C', 'D', 'E', 'F' + }; + + private final TranscriptionConfigProvider configProvider; + private final ManagedChannel originalChannel; + private final String packageName; + private final String cert; + + public TranscriptionClientFactory(Context context, TranscriptionConfigProvider configProvider) { + this(context, configProvider, getManagedChannel(configProvider)); + } + + public TranscriptionClientFactory( + Context context, TranscriptionConfigProvider configProvider, ManagedChannel managedChannel) { + this.configProvider = configProvider; + this.packageName = context.getPackageName(); + this.cert = getCertificateFingerprint(context); + originalChannel = managedChannel; + } + + public TranscriptionClient getClient() { + LogUtil.enterBlock("TranscriptionClientFactory.getClient"); + Assert.checkState(!originalChannel.isShutdown()); + Channel channel = + ClientInterceptors.intercept( + originalChannel, + new Interceptor( + packageName, cert, configProvider.getApiKey(), configProvider.getAuthToken())); + return new TranscriptionClient(VoicemailTranscriptionServiceGrpc.newBlockingStub(channel)); + } + + public void shutdown() { + LogUtil.enterBlock("TranscriptionClientFactory.shutdown"); + if (!originalChannel.isShutdown()) { + originalChannel.shutdown(); + } + } + + private static ManagedChannel getManagedChannel(TranscriptionConfigProvider configProvider) { + ManagedChannelBuilder<OkHttpChannelBuilder> builder = + OkHttpChannelBuilder.forTarget(configProvider.getServerAddress()); + // Only use plaintext for debugging + if (configProvider.shouldUsePlaintext()) { + // Just passing 'false' doesnt have the same effect as not setting this field + builder.usePlaintext(true); + } + return builder.build(); + } + + private static String getCertificateFingerprint(Context context) { + try { + PackageInfo packageInfo = + context + .getPackageManager() + .getPackageInfo(context.getPackageName(), PackageManager.GET_SIGNATURES); + if (packageInfo != null + && packageInfo.signatures != null + && packageInfo.signatures.length > 0) { + MessageDigest messageDigest = MessageDigest.getInstance(DIGEST_ALGORITHM_SHA1); + if (messageDigest == null) { + LogUtil.w( + "TranscriptionClientFactory.getCertificateFingerprint", "error getting digest."); + return null; + } + byte[] bytes = messageDigest.digest(packageInfo.signatures[0].toByteArray()); + if (bytes == null) { + LogUtil.w( + "TranscriptionClientFactory.getCertificateFingerprint", "empty message digest."); + return null; + } + + int length = bytes.length; + StringBuilder out = new StringBuilder(length * 2); + for (int i = 0; i < length; i++) { + out.append(HEX_UPPERCASE[(bytes[i] & 0xf0) >>> 4]); + out.append(HEX_UPPERCASE[bytes[i] & 0x0f]); + } + return out.toString(); + } else { + LogUtil.w( + "TranscriptionClientFactory.getCertificateFingerprint", + "failed to get package signature."); + } + } catch (Exception e) { + LogUtil.e( + "TranscriptionClientFactory.getCertificateFingerprint", + "error getting certificate fingerprint.", + e); + } + + return null; + } + + private static final class Interceptor implements ClientInterceptor { + private final String packageName; + private final String cert; + private final String apiKey; + private final String authToken; + + private static final Metadata.Key<String> API_KEY_HEADER = + Metadata.Key.of("X-Goog-Api-Key", Metadata.ASCII_STRING_MARSHALLER); + private static final Metadata.Key<String> ANDROID_PACKAGE_HEADER = + Metadata.Key.of("X-Android-Package", Metadata.ASCII_STRING_MARSHALLER); + private static final Metadata.Key<String> ANDROID_CERT_HEADER = + Metadata.Key.of("X-Android-Cert", Metadata.ASCII_STRING_MARSHALLER); + private static final Metadata.Key<String> AUTHORIZATION_HEADER = + Metadata.Key.of("authorization", Metadata.ASCII_STRING_MARSHALLER); + + public Interceptor(String packageName, String cert, String apiKey, String authToken) { + this.packageName = packageName; + this.cert = cert; + this.apiKey = apiKey; + this.authToken = authToken; + } + + @Override + public <ReqT, RespT> ClientCall<ReqT, RespT> interceptCall( + MethodDescriptor<ReqT, RespT> method, CallOptions callOptions, Channel next) { + LogUtil.enterBlock( + "TranscriptionClientFactory.interceptCall, intercepted " + method.getFullMethodName()); + ClientCall<ReqT, RespT> call = next.newCall(method, callOptions); + + call = + new ForwardingClientCall.SimpleForwardingClientCall<ReqT, RespT>(call) { + @Override + public void start(Listener<RespT> responseListener, Metadata headers) { + if (!TextUtils.isEmpty(packageName)) { + LogUtil.i( + "TranscriptionClientFactory.interceptCall", + "attaching package name: " + packageName); + headers.put(ANDROID_PACKAGE_HEADER, packageName); + } + if (!TextUtils.isEmpty(cert)) { + LogUtil.i("TranscriptionClientFactory.interceptCall", "attaching android cert"); + headers.put(ANDROID_CERT_HEADER, cert); + } + if (!TextUtils.isEmpty(apiKey)) { + LogUtil.i("TranscriptionClientFactory.interceptCall", "attaching API Key"); + headers.put(API_KEY_HEADER, apiKey); + } + if (!TextUtils.isEmpty(authToken)) { + LogUtil.i("TranscriptionClientFactory.interceptCall", "attaching auth token"); + headers.put(AUTHORIZATION_HEADER, "Bearer " + authToken); + } + super.start(responseListener, headers); + } + }; + return call; + } + } +} diff --git a/java/com/android/voicemail/impl/transcribe/grpc/TranscriptionResponse.java b/java/com/android/voicemail/impl/transcribe/grpc/TranscriptionResponse.java new file mode 100644 index 000000000..f0823de32 --- /dev/null +++ b/java/com/android/voicemail/impl/transcribe/grpc/TranscriptionResponse.java @@ -0,0 +1,53 @@ +/* + * Copyright (C) 2017 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License + */ +package com.android.voicemail.impl.transcribe.grpc; + +import android.support.annotation.Nullable; +import com.android.dialer.common.Assert; +import io.grpc.Status; + +/** + * Base class for encapulating a voicemail transcription server response. This handles the Grpc + * status response, subclasses will handle request specific responses. + */ +public abstract class TranscriptionResponse { + @Nullable public final Status status; + + TranscriptionResponse() { + this.status = null; + } + + TranscriptionResponse(Status status) { + Assert.checkArgument(status != null); + this.status = status; + } + + public boolean hasRecoverableError() { + if (status != null) { + return status.getCode() == Status.Code.UNAVAILABLE; + } + + return false; + } + + public boolean hasFatalError() { + if (status != null) { + return status.getCode() != Status.Code.OK && status.getCode() != Status.Code.UNAVAILABLE; + } + + return false; + } +} diff --git a/java/com/android/voicemail/impl/transcribe/grpc/TranscriptionResponseAsync.java b/java/com/android/voicemail/impl/transcribe/grpc/TranscriptionResponseAsync.java new file mode 100644 index 000000000..38b463053 --- /dev/null +++ b/java/com/android/voicemail/impl/transcribe/grpc/TranscriptionResponseAsync.java @@ -0,0 +1,53 @@ +/* + * Copyright (C) 2017 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License + */ +package com.android.voicemail.impl.transcribe.grpc; + +import android.support.annotation.Nullable; +import android.support.annotation.VisibleForTesting; +import com.android.dialer.common.Assert; +import com.google.internal.communications.voicemailtranscription.v1.TranscribeVoicemailAsyncResponse; +import io.grpc.Status; + +/** Container for response and status objects for an asynchronous transcription upload request */ +public class TranscriptionResponseAsync extends TranscriptionResponse { + @Nullable private final TranscribeVoicemailAsyncResponse response; + + @VisibleForTesting + public TranscriptionResponseAsync(TranscribeVoicemailAsyncResponse response) { + Assert.checkArgument(response != null); + this.response = response; + } + + @VisibleForTesting + public TranscriptionResponseAsync(Status status) { + super(status); + this.response = null; + } + + public @Nullable String getTranscriptionId() { + if (response != null) { + return response.getTranscriptionId(); + } + return null; + } + + public long getEstimatedWaitMillis() { + if (response != null) { + return response.getEstimatedWaitSecs() * 1_000L; + } + return 0; + } +} diff --git a/java/com/android/voicemail/impl/transcribe/grpc/TranscriptionResponseSync.java b/java/com/android/voicemail/impl/transcribe/grpc/TranscriptionResponseSync.java new file mode 100644 index 000000000..d2e2e218c --- /dev/null +++ b/java/com/android/voicemail/impl/transcribe/grpc/TranscriptionResponseSync.java @@ -0,0 +1,43 @@ +/* + * Copyright (C) 2017 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License + */ +package com.android.voicemail.impl.transcribe.grpc; + +import android.support.annotation.Nullable; +import android.support.annotation.VisibleForTesting; +import com.android.dialer.common.Assert; +import com.google.internal.communications.voicemailtranscription.v1.TranscribeVoicemailResponse; +import io.grpc.Status; + +/** Container for response and status objects for a synchronous transcription request */ +public class TranscriptionResponseSync extends TranscriptionResponse { + @Nullable private final TranscribeVoicemailResponse response; + + @VisibleForTesting + public TranscriptionResponseSync(Status status) { + super(status); + this.response = null; + } + + @VisibleForTesting + public TranscriptionResponseSync(TranscribeVoicemailResponse response) { + Assert.checkArgument(response != null); + this.response = response; + } + + public @Nullable String getTranscript() { + return (response != null) ? response.getTranscript() : null; + } +} diff --git a/java/com/android/voicemail/impl/transcribe/grpc/voicemail_transcription.proto b/java/com/android/voicemail/impl/transcribe/grpc/voicemail_transcription.proto new file mode 100644 index 000000000..a2064d193 --- /dev/null +++ b/java/com/android/voicemail/impl/transcribe/grpc/voicemail_transcription.proto @@ -0,0 +1,133 @@ +// LINT.IfChange + +syntax = "proto2"; + +package google.internal.communications.voicemailtranscription.v1; + +option java_multiple_files = true; +option optimize_for = LITE_RUNTIME; + +option java_package = "com.google.internal.communications.voicemailtranscription.v1"; + +// Enum that specifies supported audio formats. +enum AudioFormat { + // Default but invalid value. + AUDIO_FORMAT_UNSPECIFIED = 0; + + // Adaptive Multi-Rate Narrowband, 8kHz sampling frequency. + // https://en.wikipedia.org/wiki/Adaptive_Multi-Rate_audio_codec + AMR_NB_8KHZ = 1; +} + +// Enum that describes the status of the transcription process. +enum TranscriptionStatus { + // Default but invalid value. + TRANSCRIPTION_STATUS_UNSPECIFIED = 0; + + // Transcription was successful and the transcript is present. + SUCCESS = 1; + + // Transcription is progress. Check again later. + PENDING = 2; + + // Transcription was successful, but the expiration period has passed, which + // means that the sensative data (including the transcript) has been deleted. + // Resend the voicemail through TranscribeVoicemailAsync to retry. + EXPIRED = 3; + + // Internal error encountered during the transcription. + // Resend the voicemail through TranscribeVoicemailAsync to retry. + // This is a catch-all status for all retriable errors that aren't captured by + // a more specfic status. + FAILED_RETRY = 4; + + // Internal error encountered during the transcription. + // Do not resend the voicemail. + // This is a catch-all status for all non-retriable errors that aren't + // captured by a more specfic status. + FAILED_NO_RETRY = 5; + + // The language detected is not yet supported by this service. + // Do not resend the voicemail. + FAILED_LANGUAGE_NOT_SUPPORTED = 6; + + // No speech was detected in the voicemail. + // Do not resend the voicemail. + FAILED_NO_SPEECH_DETECTED = 7; +} + +// Request for synchronous voicemail transcription. +message TranscribeVoicemailRequest { + // Voicemail audio file containing the raw bytes we receive from the carrier. + optional bytes voicemail_data = 1; + + // Audio format of the voicemail file. + optional AudioFormat audio_format = 2; +} + +// Response for synchronous voicemail transcription. +message TranscribeVoicemailResponse { + // The transcribed text of the voicemail. + optional string transcript = 1; +} + +// Request for asynchronous voicemail transcription. +message TranscribeVoicemailAsyncRequest { + // Voicemail audio data encoded in the format specified by audio_format. + optional bytes voicemail_data = 1; + + // Audio format of the voicemail file. + optional AudioFormat audio_format = 2; +} + +// Response for asynchronous voicemail transcription containing information +// needed to fetch the transcription results through the GetTranscript method. +message TranscribeVoicemailAsyncResponse { + // Unique ID for the transcription. This ID is used for retrieving the + // voicemail transcript later. + optional string transcription_id = 1; + + // The estimated amount of time in seconds before the transcription will be + // available. + // The client should not call GetTranscript until this time has elapsed, but + // the transcript is not guaranteed to be ready by this time. + optional int64 estimated_wait_secs = 2; +} + +// Request for retrieving an asynchronously generated transcript. +message GetTranscriptRequest { + // Unique ID for the transcription. This ID was returned by + // TranscribeVoicemailAsync. + optional string transcription_id = 1; +} + +// Response for retrieving an asynchronously generated transcript. +message GetTranscriptResponse { + // Status of the trascription process. + optional TranscriptionStatus status = 1; + + // The transcribed text of the voicemail. This is only present if the status + // is SUCCESS. + optional string transcript = 2; +} + +// RPC service for transcribing voicemails. +service VoicemailTranscriptionService { + // Returns a transcript of the given voicemail. + rpc TranscribeVoicemail(TranscribeVoicemailRequest) + returns (TranscribeVoicemailResponse) {} + + // Schedules a transcription of the given voicemail. The transcript can be + // retrieved using the returned ID. + rpc TranscribeVoicemailAsync(TranscribeVoicemailAsyncRequest) + returns (TranscribeVoicemailAsyncResponse) { + } + + // Returns the transcript corresponding to the given ID, which was returned + // by TranscribeVoicemailAsync. + rpc GetTranscript(GetTranscriptRequest) returns (GetTranscriptResponse) { + } +} + +// LINT.ThenChange(//depot/google3/google/internal/communications/voicemailtranscription/v1/\ +// voicemail_transcription.proto) |