/*
* Copyright (C) 2012 The Android Open Source Project
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package com.android.test.hwuicompare;
import java.io.File;
import java.io.FileInputStream;
import java.io.FileOutputStream;
import java.io.IOException;
import java.util.ArrayList;
import java.util.Comparator;
import java.util.HashMap;
import java.util.TreeSet;
import org.json.JSONException;
import org.json.JSONObject;
import android.os.Bundle;
import android.os.Environment;
import android.os.Trace;
import android.util.Log;
import android.widget.ImageView;
import android.widget.Toast;
public class AutomaticActivity extends CompareActivity {
private static final String LOG_TAG = "AutomaticActivity";
private static final float ERROR_DISPLAY_THRESHOLD = 0.01f;
protected static final boolean DRAW_BITMAPS = false;
/**
* Threshold of error change required to consider a test regressed/improved
*/
private static final float ERROR_CHANGE_THRESHOLD = 0.001f;
private static final float[] ERROR_CUTOFFS = {
0, 0.005f, 0.01f, 0.02f, 0.05f, 0.1f, 0.25f, 0.5f, 1f, 2f
};
private final float[] mErrorRates = new float[ERROR_CUTOFFS.length];
private float mTotalTests = 0;
private float mTotalError = 0;
private int mTestsRegressed = 0;
private int mTestsImproved = 0;
private ImageView mSoftwareImageView = null;
private ImageView mHardwareImageView = null;
public abstract static class FinalCallback {
abstract void report(String name, float value);
void complete() {};
}
private final ArrayList<FinalCallback> mFinalCallbacks = new ArrayList<FinalCallback>();
Runnable mRunnable = new Runnable() {
@Override
public void run() {
loadBitmaps();
if (mSoftwareBitmap == null || mHardwareBitmap == null) {
Log.e(LOG_TAG, "bitmap is null");
return;
}
if (DRAW_BITMAPS) {
mSoftwareImageView.setImageBitmap(mSoftwareBitmap);
mHardwareImageView.setImageBitmap(mHardwareBitmap);
}
Trace.traceBegin(Trace.TRACE_TAG_ALWAYS, "calculateError");
float error = mErrorCalculator.calcErrorRS(mSoftwareBitmap, mHardwareBitmap);
Trace.traceEnd(Trace.TRACE_TAG_ALWAYS);
final String[] modifierNames = DisplayModifier.getLastAppliedModifications();
handleError(modifierNames, error);
if (DisplayModifier.step()) {
finishTest();
} else {
mHardwareView.invalidate();
if (DRAW_BITMAPS) {
mSoftwareImageView.invalidate();
mHardwareImageView.invalidate();
}
}
mHandler.removeCallbacks(mRunnable);
}
};
@Override
protected void onPause() {
super.onPause();
mHandler.removeCallbacks(mRunnable);
};
@Override
protected void onCreate(Bundle savedInstanceState) {
super.onCreate(savedInstanceState);
setContentView(R.layout.automatic_layout);
mSoftwareImageView = (ImageView) findViewById(R.id.software_image_view);
mHardwareImageView = (ImageView) findViewById(R.id.hardware_image_view);
onCreateCommon(mRunnable);
beginTest();
}
private static class TestResult {
TestResult(String label, float error) {
mLabel = label;
mTotalError = error;
mCount = 1;
}
public void addInto(float error) {
mTotalError += error;
mCount++;
}
public float getAverage() {
return mTotalError / mCount;
}
final String mLabel;
float mTotalError;
int mCount;
}
JSONObject mOutputJson = null;
JSONObject mInputJson = null;
final HashMap<String, TestResult> mModifierResults = new HashMap<String, TestResult>();
final HashMap<String, TestResult> mIndividualResults = new HashMap<String, TestResult>();
final HashMap<String, TestResult> mModifierDiffResults = new HashMap<String, TestResult>();
final HashMap<String, TestResult> mIndividualDiffResults = new HashMap<String, TestResult>();
private void beginTest() {
mFinalCallbacks.add(new FinalCallback() {
@Override
void report(String name, float value) {
Log.d(LOG_TAG, name + " " + value);
};
});
File inputFile = new File(Environment.getExternalStorageDirectory(),
"CanvasCompareInput.json");
if (inputFile.exists() && inputFile.canRead() && inputFile.length() > 0) {
try {
FileInputStream inputStream = new FileInputStream(inputFile);
Log.d(LOG_TAG, "Parsing input file...");
StringBuffer content = new StringBuffer((int)inputFile.length());
byte[] buffer = new byte[1024];
while (inputStream.read(buffer) != -1) {
content.append(new String(buffer));
}
mInputJson = new JSONObject(content.toString());
inputStream.close();
Log.d(LOG_TAG, "Parsed input file with " + mInputJson.length() + " entries");
} catch (JSONException e) {
Log.e(LOG_TAG, "error parsing input json", e);
} catch (IOException e) {
Log.e(LOG_TAG, "error reading input json from sd", e);
}
}
mOutputJson = new JSONObject();
}
private static void logTestResultHash(String label, HashMap<String, TestResult> map) {
Log.d(LOG_TAG, "---------------");
Log.d(LOG_TAG, label + ":");
Log.d(LOG_TAG, "---------------");
TreeSet<TestResult> set = new TreeSet<TestResult>(new Comparator<TestResult>() {
@Override
public int compare(TestResult lhs, TestResult rhs) {
if (lhs == rhs) return 0; // don't need to worry about complex equality
int cmp = Float.compare(lhs.getAverage(), rhs.getAverage());
if (cmp != 0) {
return cmp;
}
return lhs.mLabel.compareTo(rhs.mLabel);
}
});
for (TestResult t : map.values()) {
set.add(t);
}
for (TestResult t : set.descendingSet()) {
if (Math.abs(t.getAverage()) > ERROR_DISPLAY_THRESHOLD) {
Log.d(LOG_TAG, String.format("%2.4f : %s", t.getAverage(), t.mLabel));
}
}
Log.d(LOG_TAG, "");
}
private void finishTest() {
for (FinalCallback c : mFinalCallbacks) {
c.report("averageError", (mTotalError / mTotalTests));
for (int i = 1; i < ERROR_CUTOFFS.length; i++) {
c.report(String.format("tests with error over %1.3f", ERROR_CUTOFFS[i]),
mErrorRates[i]);
}
if (mInputJson != null) {
c.report("tests regressed", mTestsRegressed);
c.report("tests improved", mTestsImproved);
}
c.complete();
}
try {
if (mOutputJson != null) {
String outputString = mOutputJson.toString(4);
File outputFile = new File(Environment.getExternalStorageDirectory(),
"CanvasCompareOutput.json");
FileOutputStream outputStream = new FileOutputStream(outputFile);
outputStream.write(outputString.getBytes());
outputStream.close();
Log.d(LOG_TAG, "Saved output file with " + mOutputJson.length() + " entries");
}
} catch (JSONException e) {
Log.e(LOG_TAG, "error during JSON stringify", e);
} catch (IOException e) {
Log.e(LOG_TAG, "error storing JSON output on sd", e);
}
logTestResultHash("Modifier change vs previous", mModifierDiffResults);
logTestResultHash("Invidual test change vs previous", mIndividualDiffResults);
logTestResultHash("Modifier average test results", mModifierResults);
logTestResultHash("Individual test results", mIndividualResults);
Toast.makeText(getApplicationContext(), "done!", Toast.LENGTH_SHORT).show();
finish();
}
/**
* Inserts the error value into all TestResult objects, associated with each of its modifiers
*/
private static void addForAllModifiers(String fullName, float error, String[] modifierNames,
HashMap<String, TestResult> modifierResults) {
for (String modifierName : modifierNames) {
TestResult r = modifierResults.get(fullName);
if (r == null) {
modifierResults.put(modifierName, new TestResult(modifierName, error));
} else {
r.addInto(error);
}
}
}
private void handleError(final String[] modifierNames, final float error) {
String fullName = "";
for (String s : modifierNames) {
fullName = fullName.concat("." + s);
}
fullName = fullName.substring(1);
float deltaError = 0;
if (mInputJson != null) {
try {
deltaError = error - (float)mInputJson.getDouble(fullName);
} catch (JSONException e) {
Log.w(LOG_TAG, "Warning: unable to read from input json", e);
}
if (deltaError > ERROR_CHANGE_THRESHOLD) mTestsRegressed++;
if (deltaError < -ERROR_CHANGE_THRESHOLD) mTestsImproved++;
mIndividualDiffResults.put(fullName, new TestResult(fullName, deltaError));
addForAllModifiers(fullName, deltaError, modifierNames, mModifierDiffResults);
}
mIndividualResults.put(fullName, new TestResult(fullName, error));
addForAllModifiers(fullName, error, modifierNames, mModifierResults);
try {
if (mOutputJson != null) {
mOutputJson.put(fullName, error);
}
} catch (JSONException e) {
Log.e(LOG_TAG, "exception during JSON recording", e);
mOutputJson = null;
}
for (int i = 0; i < ERROR_CUTOFFS.length; i++) {
if (error <= ERROR_CUTOFFS[i]) break;
mErrorRates[i]++;
}
mTotalError += error;
mTotalTests++;
}
@Override
protected boolean forceRecreateBitmaps() {
// disable, unless needed for drawing into imageviews
return DRAW_BITMAPS;
}
// FOR TESTING
public void setFinalCallback(FinalCallback c) {
mFinalCallbacks.add(c);
}
}
|
