/*
* Copyright (C) 2011 The Android Open Source Project
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package android.filterpacks.videosrc;
import android.content.Context;
import android.content.res.AssetFileDescriptor;
import android.filterfw.core.Filter;
import android.filterfw.core.FilterContext;
import android.filterfw.core.Frame;
import android.filterfw.core.FrameFormat;
import android.filterfw.core.GenerateFieldPort;
import android.filterfw.core.GenerateFinalPort;
import android.filterfw.core.GLFrame;
import android.filterfw.core.MutableFrameFormat;
import android.filterfw.core.ShaderProgram;
import android.filterfw.format.ImageFormat;
import android.graphics.SurfaceTexture;
import android.media.MediaPlayer;
import android.net.Uri;
import android.opengl.Matrix;
import android.view.Surface;
import java.io.IOException;
import java.lang.IllegalArgumentException;
import android.util.Log;
/**
* @hide
*/
public class MediaSource extends Filter {
/** User-visible parameters */
/** The source URL for the media source. Can be an http: link to a remote
* resource, or a file: link to a local media file
*/
@GenerateFieldPort(name = "sourceUrl", hasDefault = true)
private String mSourceUrl = "";
/** An open asset file descriptor to a local media source. Default is null */
@GenerateFieldPort(name = "sourceAsset", hasDefault = true)
private AssetFileDescriptor mSourceAsset = null;
/** The context for the MediaPlayer to resolve the sourceUrl.
* Make sure this is set before the sourceUrl to avoid unexpected result.
* If the sourceUrl is not a content URI, it is OK to keep this as null. */
@GenerateFieldPort(name = "context", hasDefault = true)
private Context mContext = null;
/** Whether the media source is a URL or an asset file descriptor. Defaults
* to false.
*/
@GenerateFieldPort(name = "sourceIsUrl", hasDefault = true)
private boolean mSelectedIsUrl = false;
/** Whether the filter will always wait for a new video frame, or whether it
* will output an old frame again if a new frame isn't available. Defaults
* to true.
*/
@GenerateFinalPort(name = "waitForNewFrame", hasDefault = true)
private boolean mWaitForNewFrame = true;
/** Whether the media source should loop automatically or not. Defaults to
* true.
*/
@GenerateFieldPort(name = "loop", hasDefault = true)
private boolean mLooping = true;
/** Volume control. Currently sound is piped directly to the speakers, so
* this defaults to mute.
*/
@GenerateFieldPort(name = "volume", hasDefault = true)
private float mVolume = 0.f;
/** Orientation. This controls the output orientation of the video. Valid
* values are 0, 90, 180, 270
*/
@GenerateFieldPort(name = "orientation", hasDefault = true)
private int mOrientation = 0;
private MediaPlayer mMediaPlayer;
private GLFrame mMediaFrame;
private SurfaceTexture mSurfaceTexture;
private ShaderProgram mFrameExtractor;
private MutableFrameFormat mOutputFormat;
private int mWidth, mHeight;
// Total timeouts will be PREP_TIMEOUT*PREP_TIMEOUT_REPEAT
private static final int PREP_TIMEOUT = 100; // ms
private static final int PREP_TIMEOUT_REPEAT = 100;
private static final int NEWFRAME_TIMEOUT = 100; //ms
private static final int NEWFRAME_TIMEOUT_REPEAT = 10;
// This is an identity shader; not using the default identity
// shader because reading from a SurfaceTexture requires the
// GL_OES_EGL_image_external extension.
private final String mFrameShader =
"#extension GL_OES_EGL_image_external : require\n" +
"precision mediump float;\n" +
"uniform samplerExternalOES tex_sampler_0;\n" +
"varying vec2 v_texcoord;\n" +
"void main() {\n" +
" gl_FragColor = texture2D(tex_sampler_0, v_texcoord);\n" +
"}\n";
// The following transforms enable rotation of the decoded source.
// These are multiplied with the transform obtained from the
// SurfaceTexture to get the final transform to be set on the media source.
// Currently, given a device orientation, the MediaSource rotates in such a way
// that the source is displayed upright. A particular use case
// is "Background Replacement" feature in the Camera app
// where the MediaSource rotates the source to align with the camera feed and pass it
// on to the backdropper filter. The backdropper only does the blending
// and does not have to do any rotation
// (except for mirroring in case of front camera).
// TODO: Currently the rotations are spread over a bunch of stages in the
// pipeline. A cleaner design
// could be to cast away all the rotation in a separate filter or attach a transform
// to the frame so that MediaSource itself need not know about any rotation.
private static final float[] mSourceCoords_0 = { 1, 1, 0, 1,
0, 1, 0, 1,
1, 0, 0, 1,
0, 0, 0, 1 };
private static final float[] mSourceCoords_270 = { 0, 1, 0, 1,
0, 0, 0, 1,
1, 1, 0, 1,
1, 0, 0, 1 };
private static final float[] mSourceCoords_180 = { 0, 0, 0, 1,
1, 0, 0, 1,
0, 1, 0, 1,
1, 1, 0, 1 };
private static final float[] mSourceCoords_90 = { 1, 0, 0, 1,
1, 1, 0, 1,
0, 0, 0, 1,
0, 1, 0, 1 };
private boolean mGotSize;
private boolean mPrepared;
private boolean mPlaying;
private boolean mNewFrameAvailable;
private boolean mOrientationUpdated;
private boolean mPaused;
private boolean mCompleted;
private final boolean mLogVerbose;
private static final String TAG = "MediaSource";
public MediaSource(String name) {
super(name);
mNewFrameAvailable = false;
mLogVerbose = Log.isLoggable(TAG, Log.VERBOSE);
}
@Override
public void setupPorts() {
// Add input port
addOutputPort("video", ImageFormat.create(ImageFormat.COLORSPACE_RGBA,
FrameFormat.TARGET_GPU));
}
private void createFormats() {
mOutputFormat = ImageFormat.create(ImageFormat.COLORSPACE_RGBA,
FrameFormat.TARGET_GPU);
}
@Override
protected void prepare(FilterContext context) {
if (mLogVerbose) Log.v(TAG, "Preparing MediaSource");
mFrameExtractor = new ShaderProgram(context, mFrameShader);
// SurfaceTexture defines (0,0) to be bottom-left. The filter framework
// defines (0,0) as top-left, so do the flip here.
mFrameExtractor.setSourceRect(0, 1, 1, -1);
createFormats();
}
@Override
public void open(FilterContext context) {
if (mLogVerbose) {
Log.v(TAG, "Opening MediaSource");
if (mSelectedIsUrl) {
Log.v(TAG, "Current URL is " + mSourceUrl);
} else {
Log.v(TAG, "Current source is Asset!");
}
}
mMediaFrame = (GLFrame)context.getFrameManager().newBoundFrame(
mOutputFormat,
GLFrame.EXTERNAL_TEXTURE,
0);
mSurfaceTexture = new SurfaceTexture(mMediaFrame.getTextureId());
if (!setupMediaPlayer(mSelectedIsUrl)) {
throw new RuntimeException("Error setting up MediaPlayer!");
}
}
@Override
public void process(FilterContext context) {
// Note: process is synchronized by its caller in the Filter base class
if (mLogVerbose) Log.v(TAG, "Processing new frame");
if (mMediaPlayer == null) {
// Something went wrong in initialization or parameter updates
throw new NullPointerException("Unexpected null media player!");
}
if (mCompleted) {
// Video playback is done, so close us down
closeOutputPort("video");
return;
}
if (!mPlaying) {
int waitCount = 0;
if (mLogVerbose) Log.v(TAG, "Waiting for preparation to complete");
while (!mGotSize || !mPrepared) {
try {
this.wait(PREP_TIMEOUT);
} catch (InterruptedException e) {
// ignoring
}
if (mCompleted) {
// Video playback is done, so close us down
closeOutputPort("video");
return;
}
waitCount++;
if (waitCount == PREP_TIMEOUT_REPEAT) {
mMediaPlayer.release();
throw new RuntimeException("MediaPlayer timed out while preparing!");
}
}
if (mLogVerbose) Log.v(TAG, "Starting playback");
mMediaPlayer.start();
}
// Use last frame if paused, unless just starting playback, in which case
// we want at least one valid frame before pausing
if (!mPaused || !mPlaying) {
if (mWaitForNewFrame) {
if (mLogVerbose) Log.v(TAG, "Waiting for new frame");
int waitCount = 0;
while (!mNewFrameAvailable) {
if (waitCount == NEWFRAME_TIMEOUT_REPEAT) {
if (mCompleted) {
// Video playback is done, so close us down
closeOutputPort("video");
return;
} else {
throw new RuntimeException("Timeout waiting for new frame!");
}
}
try {
this.wait(NEWFRAME_TIMEOUT);
} catch (InterruptedException e) {
if (mLogVerbose) Log.v(TAG, "interrupted");
// ignoring
}
waitCount++;
}
mNewFrameAvailable = false;
if (mLogVerbose) Log.v(TAG, "Got new frame");
}
mSurfaceTexture.updateTexImage();
mOrientationUpdated = true;
}
if (mOrientationUpdated) {
float[] surfaceTransform = new float[16];
mSurfaceTexture.getTransformMatrix(surfaceTransform);
float[] sourceCoords = new float[16];
switch (mOrientation) {
default:
case 0:
Matrix.multiplyMM(sourceCoords, 0,
surfaceTransform, 0,
mSourceCoords_0, 0);
break;
case 90:
Matrix.multiplyMM(sourceCoords, 0,
surfaceTransform, 0,
mSourceCoords_90, 0);
break;
case 180:
Matrix.multiplyMM(sourceCoords, 0,
surfaceTransform, 0,
mSourceCoords_180, 0);
break;
case 270:
Matrix.multiplyMM(sourceCoords, 0,
surfaceTransform, 0,
mSourceCoords_270, 0);
break;
}
if (mLogVerbose) {
Log.v(TAG, "OrientationHint = " + mOrientation);
String temp = String.format("SetSourceRegion: %.2f, %.2f, %.2f, %.2f, %.2f, %.2f, %.2f, %.2f",
sourceCoords[4], sourceCoords[5],sourceCoords[0], sourceCoords[1],
sourceCoords[12], sourceCoords[13],sourceCoords[8], sourceCoords[9]);
Log.v(TAG, temp);
}
mFrameExtractor.setSourceRegion(sourceCoords[4], sourceCoords[5],
sourceCoords[0], sourceCoords[1],
sourceCoords[12], sourceCoords[13],
sourceCoords[8], sourceCoords[9]);
mOrientationUpdated = false;
}
Frame output = context.getFrameManager().newFrame(mOutputFormat);
mFrameExtractor.process(mMediaFrame, output);
long timestamp = mSurfaceTexture.getTimestamp();
if (mLogVerbose) Log.v(TAG, "Timestamp: " + (timestamp / 1000000000.0) + " s");
output.setTimestamp(timestamp);
pushOutput("video", output);
output.release();
mPlaying = true;
}
@Override
public void close(FilterContext context) {
if (mMediaPlayer.isPlaying()) {
mMediaPlayer.stop();
}
mPrepared = false;
mGotSize = false;
mPlaying = false;
mPaused = false;
mCompleted = false;
mNewFrameAvailable = false;
mMediaPlayer.release();
mMediaPlayer = null;
mSurfaceTexture.release();
mSurfaceTexture = null;
if (mLogVerbose) Log.v(TAG, "MediaSource closed");
}
@Override
public void tearDown(FilterContext context) {
if (mMediaFrame != null) {
mMediaFrame.release();
}
}
// When updating the port values of the filter, users can update sourceIsUrl to switch
// between using URL objects or Assets.
// If updating only sourceUrl/sourceAsset, MediaPlayer gets reset if the current player
// uses Url objects/Asset.
// Otherwise the new sourceUrl/sourceAsset is stored and will be used when users switch
// sourceIsUrl next time.
@Override
public void fieldPortValueUpdated(String name, FilterContext context) {
if (mLogVerbose) Log.v(TAG, "Parameter update");
if (name.equals("sourceUrl")) {
if (isOpen()) {
if (mLogVerbose) Log.v(TAG, "Opening new source URL");
if (mSelectedIsUrl) {
setupMediaPlayer(mSelectedIsUrl);
}
}
} else if (name.equals("sourceAsset") ) {
if (isOpen()) {
if (mLogVerbose) Log.v(TAG, "Opening new source FD");
if (!mSelectedIsUrl) {
setupMediaPlayer(mSelectedIsUrl);
}
}
} else if (name.equals("loop")) {
if (isOpen()) {
mMediaPlayer.setLooping(mLooping);
}
} else if (name.equals("sourceIsUrl")) {
if (isOpen()){
if (mSelectedIsUrl){
if (mLogVerbose) Log.v(TAG, "Opening new source URL");
} else {
if (mLogVerbose) Log.v(TAG, "Opening new source Asset");
}
setupMediaPlayer(mSelectedIsUrl);
}
} else if (name.equals("volume")) {
if (isOpen()) {
mMediaPlayer.setVolume(mVolume, mVolume);
}
} else if (name.equals("orientation") && mGotSize) {
if (mOrientation == 0 || mOrientation == 180) {
mOutputFormat.setDimensions(mWidth, mHeight);
} else {
mOutputFormat.setDimensions(mHeight, mWidth);
}
mOrientationUpdated = true;
}
}
synchronized public void pauseVideo(boolean pauseState) {
if (isOpen()) {
if (pauseState && !mPaused) {
mMediaPlayer.pause();
} else if (!pauseState && mPaused) {
mMediaPlayer.start();
}
}
mPaused = pauseState;
}
/** Creates a media player, sets it up, and calls prepare */
synchronized private boolean setupMediaPlayer(boolean useUrl) {
mPrepared = false;
mGotSize = false;
mPlaying = false;
mPaused = false;
mCompleted = false;
mNewFrameAvailable = false;
if (mLogVerbose) Log.v(TAG, "Setting up playback.");
if (mMediaPlayer != null) {
// Clean up existing media players
if (mLogVerbose) Log.v(TAG, "Resetting existing MediaPlayer.");
mMediaPlayer.reset();
} else {
// Create new media player
if (mLogVerbose) Log.v(TAG, "Creating new MediaPlayer.");
mMediaPlayer = new MediaPlayer();
}
if (mMediaPlayer == null) {
throw new RuntimeException("Unable to create a MediaPlayer!");
}
// Set up data sources, etc
try {
if (useUrl) {
if (mLogVerbose) Log.v(TAG, "Setting MediaPlayer source to URI " + mSourceUrl);
if (mContext == null) {
mMediaPlayer.setDataSource(mSourceUrl);
} else {
mMediaPlayer.setDataSource(mContext, Uri.parse(mSourceUrl.toString()));
}
} else {
if (mLogVerbose) Log.v(TAG, "Setting MediaPlayer source to asset " + mSourceAsset);
mMediaPlayer.setDataSource(mSourceAsset.getFileDescriptor(), mSourceAsset.getStartOffset(), mSourceAsset.getLength());
}
} catch(IOException e) {
mMediaPlayer.release();
mMediaPlayer = null;
if (useUrl) {
throw new RuntimeException(String.format("Unable to set MediaPlayer to URL %s!", mSourceUrl), e);
} else {
throw new RuntimeException(String.format("Unable to set MediaPlayer to asset %s!", mSourceAsset), e);
}
} catch(IllegalArgumentException e) {
mMediaPlayer.release();
mMediaPlayer = null;
if (useUrl) {
throw new RuntimeException(String.format("Unable to set MediaPlayer to URL %s!", mSourceUrl), e);
} else {
throw new RuntimeException(String.format("Unable to set MediaPlayer to asset %s!", mSourceAsset), e);
}
}
mMediaPlayer.setLooping(mLooping);
mMediaPlayer.setVolume(mVolume, mVolume);
// Bind it to our media frame
Surface surface = new Surface(mSurfaceTexture);
mMediaPlayer.setSurface(surface);
surface.release();
// Connect Media Player to callbacks
mMediaPlayer.setOnVideoSizeChangedListener(onVideoSizeChangedListener);
mMediaPlayer.setOnPreparedListener(onPreparedListener);
mMediaPlayer.setOnCompletionListener(onCompletionListener);
// Connect SurfaceTexture to callback
mSurfaceTexture.setOnFrameAvailableListener(onMediaFrameAvailableListener);
if (mLogVerbose) Log.v(TAG, "Preparing MediaPlayer.");
mMediaPlayer.prepareAsync();
return true;
}
private MediaPlayer.OnVideoSizeChangedListener onVideoSizeChangedListener =
new MediaPlayer.OnVideoSizeChangedListener() {
public void onVideoSizeChanged(MediaPlayer mp, int width, int height) {
if (mLogVerbose) Log.v(TAG, "MediaPlayer sent dimensions: " + width + " x " + height);
if (!mGotSize) {
if (mOrientation == 0 || mOrientation == 180) {
mOutputFormat.setDimensions(width, height);
} else {
mOutputFormat.setDimensions(height, width);
}
mWidth = width;
mHeight = height;
} else {
if (mOutputFormat.getWidth() != width ||
mOutputFormat.getHeight() != height) {
Log.e(TAG, "Multiple video size change events received!");
}
}
synchronized(MediaSource.this) {
mGotSize = true;
MediaSource.this.notify();
}
}
};
private MediaPlayer.OnPreparedListener onPreparedListener =
new MediaPlayer.OnPreparedListener() {
public void onPrepared(MediaPlayer mp) {
if (mLogVerbose) Log.v(TAG, "MediaPlayer is prepared");
synchronized(MediaSource.this) {
mPrepared = true;
MediaSource.this.notify();
}
}
};
private MediaPlayer.OnCompletionListener onCompletionListener =
new MediaPlayer.OnCompletionListener() {
public void onCompletion(MediaPlayer mp) {
if (mLogVerbose) Log.v(TAG, "MediaPlayer has completed playback");
synchronized(MediaSource.this) {
mCompleted = true;
}
}
};
private SurfaceTexture.OnFrameAvailableListener onMediaFrameAvailableListener =
new SurfaceTexture.OnFrameAvailableListener() {
public void onFrameAvailable(SurfaceTexture surfaceTexture) {
if (mLogVerbose) Log.v(TAG, "New frame from media player");
synchronized(MediaSource.this) {
if (mLogVerbose) Log.v(TAG, "New frame: notify");
mNewFrameAvailable = true;
MediaSource.this.notify();
if (mLogVerbose) Log.v(TAG, "New frame: notify done");
}
}
};
}
|
