diff --git a/android/app/src/main/kotlin/com/hmg/hmgDr/ui/fragment/VideoCallFragment.kt b/android/app/src/main/kotlin/com/hmg/hmgDr/ui/fragment/VideoCallFragment.kt
index 833855a0..5b98b27d 100644
--- a/android/app/src/main/kotlin/com/hmg/hmgDr/ui/fragment/VideoCallFragment.kt
+++ b/android/app/src/main/kotlin/com/hmg/hmgDr/ui/fragment/VideoCallFragment.kt
@@ -27,6 +27,8 @@ import com.hmg.hmgDr.ui.VideoCallContract.VideoCallPresenter
import com.hmg.hmgDr.ui.VideoCallContract.VideoCallView
import com.hmg.hmgDr.ui.VideoCallPresenterImpl
import com.hmg.hmgDr.ui.VideoCallResponseListener
+import com.hmg.hmgDr.util.DynamicVideoRenderer
+import com.hmg.hmgDr.util.ThumbnailCircleVideoRenderer
import com.opentok.android.*
import com.opentok.android.PublisherKit.PublisherListener
import pub.devrel.easypermissions.AfterPermissionGranted
@@ -60,6 +62,7 @@ class VideoCallFragment : DialogFragment(), PermissionCallbacks, Session.Session
private var mVolRunnable: Runnable? = null
private var mConnectedRunnable: Runnable? = null
+ private lateinit var thumbnail_container: FrameLayout
private lateinit var mPublisherViewContainer: FrameLayout
private lateinit var mPublisherViewIcon: View
private lateinit var mSubscriberViewContainer: FrameLayout
@@ -220,6 +223,7 @@ class VideoCallFragment : DialogFragment(), PermissionCallbacks, Session.Session
layoutName = view.findViewById(R.id.layout_name)
layoutMini = view.findViewById(R.id.layout_mini)
icMini = view.findViewById(R.id.ic_mini)
+ thumbnail_container = view.findViewById(R.id.thumbnail_container)
mPublisherViewContainer = view.findViewById(R.id.local_video_view_container)
mPublisherViewIcon = view.findViewById(R.id.local_video_view_icon)
mSubscriberViewIcon = view.findViewById(R.id.remote_video_view_icon)
@@ -345,12 +349,16 @@ class VideoCallFragment : DialogFragment(), PermissionCallbacks, Session.Session
override fun onConnected(session: Session?) {
Log.i(TAG, "Session Connected")
- mPublisher = Publisher.Builder(requireContext()).build()
+ mPublisher = Publisher.Builder(requireContext())
+// .name("publisher")
+// .renderer(ThumbnailCircleVideoRenderer(requireContext()))
+ .build()
mPublisher!!.setPublisherListener(this)
- mPublisherViewContainer.addView(mPublisher!!.view)
- if (mPublisher!!.getView() is GLSurfaceView) {
- (mPublisher!!.getView() as GLSurfaceView).setZOrderOnTop(true)
+ if (mPublisher!!.view is GLSurfaceView) {
+ (mPublisher!!.view as GLSurfaceView).setZOrderOnTop(true)
}
+
+ mPublisherViewContainer.addView(mPublisher!!.view)
mSession!!.publish(mPublisher)
if (!resume) {
@@ -368,7 +376,7 @@ class VideoCallFragment : DialogFragment(), PermissionCallbacks, Session.Session
override fun onError(session: Session, opentokError: OpentokError) {
Log.d(TAG, "onError: Error (" + opentokError.message + ") in session " + session.sessionId)
- // videoCallResponseListener?.errorHandle("Error (" + opentokError.message + ") in session ")
+ // videoCallResponseListener?.errorHandle("Error (" + opentokError.message + ") in session ")
// dialog?.dismiss()
}
@@ -391,7 +399,7 @@ class VideoCallFragment : DialogFragment(), PermissionCallbacks, Session.Session
return
}
if (mSubscriber!!.stream == stream) {
- mSubscriberViewContainer!!.removeView(mSubscriber!!.view)
+ mSubscriberViewContainer.removeView(mSubscriber!!.view)
mSubscriber!!.destroy()
mSubscriber = null
}
@@ -408,13 +416,49 @@ class VideoCallFragment : DialogFragment(), PermissionCallbacks, Session.Session
override fun onError(publisherKit: PublisherKit?, opentokError: OpentokError) {
Log.d(VideoCallFragment.TAG, "onError: Error (" + opentokError.message + ") in publisher")
- // videoCallResponseListener?.errorHandle("Error (" + opentokError.message + ") in publisher")
+ // videoCallResponseListener?.errorHandle("Error (" + opentokError.message + ") in publisher")
// dialog?.dismiss()
}
override fun onVideoDataReceived(subscriberKit: SubscriberKit?) {
mSubscriber!!.setStyle(BaseVideoRenderer.STYLE_VIDEO_SCALE, BaseVideoRenderer.STYLE_VIDEO_FILL)
- mSubscriberViewContainer!!.addView(mSubscriber!!.view)
+ (mSubscriber!!.renderer as DynamicVideoRenderer).enableThumbnailCircle(false)
+ mSubscriberViewContainer.addView(mSubscriber!!.view)
+// switchToThumbnailCircle()
+ }
+
+ fun switchToThumbnailCircle() {
+ thumbnail_container.postDelayed({
+ val view = mSubscriber!!.view
+ if (view.parent != null) {
+ (view.parent as ViewGroup).removeView(view)
+ }
+ if (view is GLSurfaceView) {
+ view.setZOrderOnTop(true)
+ if (mSubscriber!!.renderer is DynamicVideoRenderer) {
+ (mSubscriber!!.renderer as DynamicVideoRenderer).enableThumbnailCircle(true)
+ thumbnail_container.addView(view)
+ }
+ }
+ switchToFullScreenView()
+ }, 4000)
+ }
+
+ fun switchToFullScreenView() {
+ mSubscriberViewContainer.postDelayed({
+ val view = mSubscriber!!.view
+ if (view.parent != null) {
+ (view.parent as ViewGroup).removeView(view)
+ }
+ if (view is GLSurfaceView) {
+ view.setZOrderOnTop(false)
+ if (mSubscriber!!.renderer is DynamicVideoRenderer) {
+ (mSubscriber!!.renderer as DynamicVideoRenderer).enableThumbnailCircle(false)
+ mSubscriberViewContainer.addView(view)
+ }
+ }
+ switchToThumbnailCircle()
+ }, 4000)
}
override fun onVideoDisabled(subscriberKit: SubscriberKit?, s: String?) {}
@@ -426,7 +470,9 @@ class VideoCallFragment : DialogFragment(), PermissionCallbacks, Session.Session
override fun onVideoDisableWarningLifted(subscriberKit: SubscriberKit?) {}
private fun subscribeToStream(stream: Stream) {
- mSubscriber = Subscriber.Builder(requireContext(), stream).build()
+ mSubscriber = Subscriber.Builder(requireContext(), stream)
+ .renderer(DynamicVideoRenderer(requireContext()))
+ .build()
mSubscriber!!.setVideoListener(this)
mSession!!.subscribe(mSubscriber)
}
@@ -488,31 +534,29 @@ class VideoCallFragment : DialogFragment(), PermissionCallbacks, Session.Session
disconnectSession()
}
- private fun miniCircleDoubleTap(){
- if (isCircle){
+ private fun miniCircleDoubleTap() {
+ if (isCircle) {
onMiniCircleClicked()
}
}
- private fun onMiniCircleClicked(){
+ private fun onMiniCircleClicked() {
if (isCircle) {
dialog?.window?.setLayout(
400,
600
)
- videoCallContainer.setBackgroundColor(ContextCompat.getColor(requireContext(), R.color.text_color))
- mSubscriberViewContainer.setBackgroundColor(ContextCompat.getColor(requireContext(), R.color.remoteBackground))
+ (mSubscriber!!.renderer as DynamicVideoRenderer).enableThumbnailCircle(false)
} else {
dialog?.window?.setLayout(
- 200,
- 200
+ 300,
+ 300
)
- videoCallContainer.background = ContextCompat.getDrawable(requireContext(), R.drawable.circle_shape)
- mSubscriberViewContainer.background = ContextCompat.getDrawable(requireContext(), R.drawable.circle_shape)
+ (mSubscriber!!.renderer as DynamicVideoRenderer).enableThumbnailCircle(true)
}
isCircle = !isCircle
- if(isCircle){
+ if (isCircle) {
controlPanel.visibility = View.GONE
layoutMini.visibility = View.GONE
} else {
@@ -548,16 +592,16 @@ class VideoCallFragment : DialogFragment(), PermissionCallbacks, Session.Session
val btnMinimizeLayoutParam: ConstraintLayout.LayoutParams = btnMinimize.layoutParams as ConstraintLayout.LayoutParams
val mCallBtnLayoutParam: ConstraintLayout.LayoutParams = mCallBtn.layoutParams as ConstraintLayout.LayoutParams
- val localPreviewMargin : Int = context!!.resources.getDimension(R.dimen.local_preview_margin_top).toInt()
- val localPreviewWidth : Int = context!!.resources.getDimension(R.dimen.local_preview_width).toInt()
- val localPreviewHeight : Int = context!!.resources.getDimension(R.dimen.local_preview_height).toInt()
+ val localPreviewMargin: Int = context!!.resources.getDimension(R.dimen.local_preview_margin_top).toInt()
+ val localPreviewWidth: Int = context!!.resources.getDimension(R.dimen.local_preview_width).toInt()
+ val localPreviewHeight: Int = context!!.resources.getDimension(R.dimen.local_preview_height).toInt()
// val localPreviewIconSize: Int = context!!.resources.getDimension(R.dimen.local_back_icon_size).toInt()
// val localPreviewMarginSmall : Int = context!!.resources.getDimension(R.dimen.local_preview_margin_small).toInt()
// val localPreviewWidthSmall : Int = context!!.resources.getDimension(R.dimen.local_preview_width_small).toInt()
// val localPreviewHeightSmall : Int = context!!.resources.getDimension(R.dimen.local_preview_height_small).toInt()
// val localPreviewIconSmall: Int = context!!.resources.getDimension(R.dimen.local_back_icon_size_small).toInt()
// val localPreviewLayoutIconParam : FrameLayout.LayoutParams
- val localPreviewLayoutParam : RelativeLayout.LayoutParams = mPublisherViewContainer.layoutParams as RelativeLayout.LayoutParams
+ val localPreviewLayoutParam: RelativeLayout.LayoutParams = mPublisherViewContainer.layoutParams as RelativeLayout.LayoutParams
val remotePreviewIconSize: Int = context!!.resources.getDimension(R.dimen.remote_back_icon_size).toInt()
val remotePreviewIconSizeSmall: Int = context!!.resources.getDimension(R.dimen.remote_back_icon_size_small).toInt()
@@ -611,7 +655,7 @@ class VideoCallFragment : DialogFragment(), PermissionCallbacks, Session.Session
remotePreviewLayoutParam.width = remotePreviewIconSizeSmall
remotePreviewLayoutParam.height = remotePreviewIconSizeSmall
- if(isCircle){
+ if (isCircle) {
controlPanel.visibility = View.GONE
layoutMini.visibility = View.GONE
} else {
@@ -651,7 +695,7 @@ class VideoCallFragment : DialogFragment(), PermissionCallbacks, Session.Session
isSpeckerClicked = !isSpeckerClicked
mSubscriber!!.subscribeToAudio = !isSpeckerClicked
val res = if (isSpeckerClicked) R.drawable.audio_disabled else R.drawable.audio_enabled
- mspeckerBtn!!.setImageResource(res)
+ mspeckerBtn.setImageResource(res)
}
}
@@ -722,7 +766,7 @@ class VideoCallFragment : DialogFragment(), PermissionCallbacks, Session.Session
}
private fun showControlPanelTemporarily() {
- if (!isCircle){
+ if (!isCircle) {
controlPanel.visibility = View.VISIBLE
mVolHandler!!.removeCallbacks(mVolRunnable!!)
mVolHandler!!.postDelayed(mVolRunnable!!, (5 * 1000).toLong())
diff --git a/android/app/src/main/kotlin/com/hmg/hmgDr/util/DynamicVideoRenderer.kt b/android/app/src/main/kotlin/com/hmg/hmgDr/util/DynamicVideoRenderer.kt
new file mode 100644
index 00000000..1a307eb5
--- /dev/null
+++ b/android/app/src/main/kotlin/com/hmg/hmgDr/util/DynamicVideoRenderer.kt
@@ -0,0 +1,379 @@
+package com.hmg.hmgDr.util
+
+import android.content.Context
+import android.content.res.Resources
+import android.graphics.PixelFormat
+import android.opengl.GLES20
+import android.opengl.GLSurfaceView
+import android.opengl.Matrix
+import android.view.View
+import com.opentok.android.BaseVideoRenderer
+import java.nio.ByteBuffer
+import java.nio.ByteOrder
+import java.nio.FloatBuffer
+import java.nio.ShortBuffer
+import java.util.concurrent.locks.ReentrantLock
+import javax.microedition.khronos.egl.EGLConfig
+import javax.microedition.khronos.opengles.GL10
+
+/*
+* https://nhancv.medium.com/android-how-to-make-a-circular-view-as-a-thumbnail-of-opentok-27992aee15c9
+* to solve make circle video stream
+* */
+
+class DynamicVideoRenderer(private val mContext: Context) : BaseVideoRenderer() {
+ private val mView: GLSurfaceView = GLSurfaceView(mContext)
+ private val mRenderer: MyRenderer
+
+ interface DynamicVideoRendererMetadataListener {
+ fun onMetadataReady(metadata: ByteArray?)
+ }
+
+ fun setDynamicVideoRendererMetadataListener(metadataListener: DynamicVideoRendererMetadataListener?) {
+ mRenderer.metadataListener = metadataListener
+ }
+
+ fun enableThumbnailCircle(enable: Boolean) {
+ mRenderer.requestEnableThumbnailCircle = enable
+ }
+
+ internal class MyRenderer : GLSurfaceView.Renderer {
+ var mTextureIds = IntArray(3)
+ var mScaleMatrix = FloatArray(16)
+ private val mVertexBuffer: FloatBuffer
+ private val mTextureBuffer: FloatBuffer
+ private val mDrawListBuffer: ShortBuffer
+ var requestEnableThumbnailCircle = false
+ var mVideoFitEnabled = true
+ var mVideoDisabled = false
+ private val mVertexIndex = shortArrayOf(0, 1, 2, 0, 2, 3) // order to draw
+
+ // vertices
+ private val vertexShaderCode = """uniform mat4 uMVPMatrix;attribute vec4 aPosition;
+attribute vec2 aTextureCoord;
+varying vec2 vTextureCoord;
+void main() {
+ gl_Position = uMVPMatrix * aPosition;
+ vTextureCoord = aTextureCoord;
+}
+"""
+ private val fragmentShaderCode = """precision mediump float;
+uniform sampler2D Ytex;
+uniform sampler2D Utex,Vtex;
+uniform int enableCircle;
+uniform vec2 radiusDp;
+varying vec2 vTextureCoord;
+void main(void) {
+ float nx,ny,r,g,b,y,u,v;
+ mediump vec4 txl,ux,vx; nx=vTextureCoord[0];
+ ny=vTextureCoord[1];
+ y=texture2D(Ytex,vec2(nx,ny)).r;
+ u=texture2D(Utex,vec2(nx,ny)).r;
+ v=texture2D(Vtex,vec2(nx,ny)).r;
+ y=1.1643*(y-0.0625);
+ u=u-0.5;
+ v=v-0.5;
+ r=y+1.5958*v;
+ g=y-0.39173*u-0.81290*v;
+ b=y+2.017*u;
+ if (enableCircle > 0) {
+ float radius = 0.5;
+ vec4 color0 = vec4(0.0, 0.0, 0.0, 0.0);
+ vec4 color1 = vec4(r, g, b, 1.0);
+ vec2 st = (gl_FragCoord.xy/radiusDp.xy); float dist = radius - distance(st,vec2(0.5));
+ float t = 1.0;
+ if (dist < 0.0) t = 0.0;
+ gl_FragColor = mix(color0, color1, t);
+ }
+ else {
+ gl_FragColor = vec4(r, g, b, 1.0);
+ }
+}
+"""
+ var mFrameLock = ReentrantLock()
+ var mCurrentFrame: Frame? = null
+ private var mProgram = 0
+ private var mTextureWidth = 0
+ private var mTextureHeight = 0
+ private var mViewportWidth = 0
+ private var mViewportHeight = 0
+ override fun onSurfaceCreated(gl: GL10, config: EGLConfig) {
+ gl.glClearColor(0f, 0f, 0f, 1f)
+ GLES20.glClear(GLES20.GL_COLOR_BUFFER_BIT)
+ val vertexShader = loadShader(GLES20.GL_VERTEX_SHADER,
+ vertexShaderCode)
+ val fragmentShader = loadShader(GLES20.GL_FRAGMENT_SHADER,
+ fragmentShaderCode)
+ mProgram = GLES20.glCreateProgram() // create empty OpenGL ES
+ // Program
+ GLES20.glAttachShader(mProgram, vertexShader) // add the vertex
+ // shader to program
+ GLES20.glAttachShader(mProgram, fragmentShader) // add the fragment
+ // shader to
+ // program
+ GLES20.glLinkProgram(mProgram)
+ val positionHandle = GLES20.glGetAttribLocation(mProgram,
+ "aPosition")
+ val textureHandle = GLES20.glGetAttribLocation(mProgram,
+ "aTextureCoord")
+ GLES20.glVertexAttribPointer(positionHandle, COORDS_PER_VERTEX,
+ GLES20.GL_FLOAT, false, COORDS_PER_VERTEX * 4,
+ mVertexBuffer)
+ GLES20.glEnableVertexAttribArray(positionHandle)
+ GLES20.glVertexAttribPointer(textureHandle,
+ TEXTURECOORDS_PER_VERTEX, GLES20.GL_FLOAT, false,
+ TEXTURECOORDS_PER_VERTEX * 4, mTextureBuffer)
+ GLES20.glEnableVertexAttribArray(textureHandle)
+ GLES20.glUseProgram(mProgram)
+ var i = GLES20.glGetUniformLocation(mProgram, "Ytex")
+ GLES20.glUniform1i(i, 0) /* Bind Ytex to texture unit 0 */
+ i = GLES20.glGetUniformLocation(mProgram, "Utex")
+ GLES20.glUniform1i(i, 1) /* Bind Utex to texture unit 1 */
+ i = GLES20.glGetUniformLocation(mProgram, "Vtex")
+ GLES20.glUniform1i(i, 2) /* Bind Vtex to texture unit 2 */
+ val radiusDpLocation = GLES20.glGetUniformLocation(mProgram, "radiusDp")
+ val radiusDp = (Resources.getSystem().displayMetrics.density * THUMBNAIL_SIZE).toInt()
+ GLES20.glUniform2f(radiusDpLocation, radiusDp.toFloat(), radiusDp.toFloat())
+ mTextureWidth = 0
+ mTextureHeight = 0
+ }
+
+ fun enableThumbnailCircle(enable: Boolean) {
+ GLES20.glUseProgram(mProgram)
+ val enableCircleLocation = GLES20.glGetUniformLocation(mProgram, "enableCircle")
+ GLES20.glUniform1i(enableCircleLocation, if (enable) 1 else 0)
+ }
+
+ fun setupTextures(frame: Frame) {
+ if (mTextureIds[0] != 0) {
+ GLES20.glDeleteTextures(3, mTextureIds, 0)
+ }
+ GLES20.glGenTextures(3, mTextureIds, 0)
+ val w = frame.width
+ val h = frame.height
+ val hw = w + 1 shr 1
+ val hh = h + 1 shr 1
+ initializeTexture(GLES20.GL_TEXTURE0, mTextureIds[0], w, h)
+ initializeTexture(GLES20.GL_TEXTURE1, mTextureIds[1], hw, hh)
+ initializeTexture(GLES20.GL_TEXTURE2, mTextureIds[2], hw, hh)
+ mTextureWidth = frame.width
+ mTextureHeight = frame.height
+ }
+
+ fun updateTextures(frame: Frame) {
+ val width = frame.width
+ val height = frame.height
+ val half_width = width + 1 shr 1
+ val half_height = height + 1 shr 1
+ val y_size = width * height
+ val uv_size = half_width * half_height
+ val bb = frame.buffer
+ // If we are reusing this frame, make sure we reset position and
+ // limit
+ bb.clear()
+ if (bb.remaining() == y_size + uv_size * 2) {
+ bb.position(0)
+ GLES20.glPixelStorei(GLES20.GL_UNPACK_ALIGNMENT, 1)
+ GLES20.glPixelStorei(GLES20.GL_PACK_ALIGNMENT, 1)
+ GLES20.glActiveTexture(GLES20.GL_TEXTURE0)
+ GLES20.glBindTexture(GLES20.GL_TEXTURE_2D, mTextureIds[0])
+ GLES20.glTexSubImage2D(GLES20.GL_TEXTURE_2D, 0, 0, 0, width,
+ height, GLES20.GL_LUMINANCE, GLES20.GL_UNSIGNED_BYTE,
+ bb)
+ bb.position(y_size)
+ GLES20.glActiveTexture(GLES20.GL_TEXTURE1)
+ GLES20.glBindTexture(GLES20.GL_TEXTURE_2D, mTextureIds[1])
+ GLES20.glTexSubImage2D(GLES20.GL_TEXTURE_2D, 0, 0, 0,
+ half_width, half_height, GLES20.GL_LUMINANCE,
+ GLES20.GL_UNSIGNED_BYTE, bb)
+ bb.position(y_size + uv_size)
+ GLES20.glActiveTexture(GLES20.GL_TEXTURE2)
+ GLES20.glBindTexture(GLES20.GL_TEXTURE_2D, mTextureIds[2])
+ GLES20.glTexSubImage2D(GLES20.GL_TEXTURE_2D, 0, 0, 0,
+ half_width, half_height, GLES20.GL_LUMINANCE,
+ GLES20.GL_UNSIGNED_BYTE, bb)
+ } else {
+ mTextureWidth = 0
+ mTextureHeight = 0
+ }
+ }
+
+ override fun onSurfaceChanged(gl: GL10, width: Int, height: Int) {
+ GLES20.glViewport(0, 0, width, height)
+ mViewportWidth = width
+ mViewportHeight = height
+ }
+
+ var metadataListener: DynamicVideoRendererMetadataListener? = null
+ override fun onDrawFrame(gl: GL10) {
+ gl.glClearColor(0f, 0f, 0f, 0f)
+ GLES20.glClear(GLES20.GL_COLOR_BUFFER_BIT)
+ mFrameLock.lock()
+ if (mCurrentFrame != null && !mVideoDisabled) {
+ GLES20.glUseProgram(mProgram)
+ if (mTextureWidth != mCurrentFrame!!.width
+ || mTextureHeight != mCurrentFrame!!.height) {
+ setupTextures(mCurrentFrame!!)
+ }
+ updateTextures(mCurrentFrame!!)
+ Matrix.setIdentityM(mScaleMatrix, 0)
+ var scaleX = 1.0f
+ var scaleY = 1.0f
+ val ratio = (mCurrentFrame!!.width.toFloat()
+ / mCurrentFrame!!.height)
+ val vratio = mViewportWidth.toFloat() / mViewportHeight
+ if (mVideoFitEnabled) {
+ if (ratio > vratio) {
+ scaleY = vratio / ratio
+ } else {
+ scaleX = ratio / vratio
+ }
+ } else {
+ if (ratio < vratio) {
+ scaleY = vratio / ratio
+ } else {
+ scaleX = ratio / vratio
+ }
+ }
+ Matrix.scaleM(mScaleMatrix, 0,
+ scaleX * if (mCurrentFrame!!.isMirroredX) -1.0f else 1.0f,
+ scaleY, 1f)
+ metadataListener?.onMetadataReady(mCurrentFrame!!.metadata)
+ val mMVPMatrixHandle = GLES20.glGetUniformLocation(mProgram,
+ "uMVPMatrix")
+ GLES20.glUniformMatrix4fv(mMVPMatrixHandle, 1, false,
+ mScaleMatrix, 0)
+ enableThumbnailCircle(requestEnableThumbnailCircle)
+ GLES20.glDrawElements(GLES20.GL_TRIANGLES, mVertexIndex.size,
+ GLES20.GL_UNSIGNED_SHORT, mDrawListBuffer)
+ } else {
+ //black frame when video is disabled
+ gl.glClearColor(0f, 0f, 0f, 1f)
+ GLES20.glClear(GLES20.GL_COLOR_BUFFER_BIT)
+ }
+ mFrameLock.unlock()
+ }
+
+ fun displayFrame(frame: Frame?) {
+ mFrameLock.lock()
+ if (mCurrentFrame != null) {
+ mCurrentFrame!!.recycle()
+ }
+ mCurrentFrame = frame
+ mFrameLock.unlock()
+ }
+
+ fun disableVideo(b: Boolean) {
+ mFrameLock.lock()
+ mVideoDisabled = b
+ if (mVideoDisabled) {
+ if (mCurrentFrame != null) {
+ mCurrentFrame!!.recycle()
+ }
+ mCurrentFrame = null
+ }
+ mFrameLock.unlock()
+ }
+
+ fun enableVideoFit(enableVideoFit: Boolean) {
+ mVideoFitEnabled = enableVideoFit
+ }
+
+ companion object {
+ // number of coordinates per vertex in this array
+ const val COORDS_PER_VERTEX = 3
+ const val TEXTURECOORDS_PER_VERTEX = 2
+ var mXYZCoords = floatArrayOf(
+ -1.0f, 1.0f, 0.0f, // top left
+ -1.0f, -1.0f, 0.0f, // bottom left
+ 1.0f, -1.0f, 0.0f, // bottom right
+ 1.0f, 1.0f, 0.0f // top right
+ )
+ var mUVCoords = floatArrayOf(0f, 0f, 0f, 1f, 1f, 1f, 1f, 0f)
+ fun initializeTexture(name: Int, id: Int, width: Int, height: Int) {
+ GLES20.glActiveTexture(name)
+ GLES20.glBindTexture(GLES20.GL_TEXTURE_2D, id)
+ GLES20.glTexParameterf(GLES20.GL_TEXTURE_2D,
+ GLES20.GL_TEXTURE_MIN_FILTER, GLES20.GL_NEAREST.toFloat())
+ GLES20.glTexParameterf(GLES20.GL_TEXTURE_2D,
+ GLES20.GL_TEXTURE_MAG_FILTER, GLES20.GL_LINEAR.toFloat())
+ GLES20.glTexParameterf(GLES20.GL_TEXTURE_2D,
+ GLES20.GL_TEXTURE_WRAP_S, GLES20.GL_CLAMP_TO_EDGE.toFloat())
+ GLES20.glTexParameterf(GLES20.GL_TEXTURE_2D,
+ GLES20.GL_TEXTURE_WRAP_T, GLES20.GL_CLAMP_TO_EDGE.toFloat())
+ GLES20.glTexImage2D(GLES20.GL_TEXTURE_2D, 0, GLES20.GL_LUMINANCE,
+ width, height, 0, GLES20.GL_LUMINANCE,
+ GLES20.GL_UNSIGNED_BYTE, null)
+ }
+
+ fun loadShader(type: Int, shaderCode: String?): Int {
+ val shader = GLES20.glCreateShader(type)
+ GLES20.glShaderSource(shader, shaderCode)
+ GLES20.glCompileShader(shader)
+ return shader
+ }
+ }
+
+ init {
+ val bb = ByteBuffer.allocateDirect(mXYZCoords.size * 4)
+ bb.order(ByteOrder.nativeOrder())
+ mVertexBuffer = bb.asFloatBuffer()
+ mVertexBuffer.put(mXYZCoords)
+ mVertexBuffer.position(0)
+ val tb = ByteBuffer.allocateDirect(mUVCoords.size * 4)
+ tb.order(ByteOrder.nativeOrder())
+ mTextureBuffer = tb.asFloatBuffer()
+ mTextureBuffer.put(mUVCoords)
+ mTextureBuffer.position(0)
+ val dlb = ByteBuffer.allocateDirect(mVertexIndex.size * 2)
+ dlb.order(ByteOrder.nativeOrder())
+ mDrawListBuffer = dlb.asShortBuffer()
+ mDrawListBuffer.put(mVertexIndex)
+ mDrawListBuffer.position(0)
+ }
+ }
+
+ override fun onFrame(frame: Frame) {
+ mRenderer.displayFrame(frame)
+ mView.requestRender()
+ }
+
+ override fun setStyle(key: String, value: String) {
+ if (STYLE_VIDEO_SCALE == key) {
+ if (STYLE_VIDEO_FIT == value) {
+ mRenderer.enableVideoFit(true)
+ } else if (STYLE_VIDEO_FILL == value) {
+ mRenderer.enableVideoFit(false)
+ }
+ }
+ }
+
+ override fun onVideoPropertiesChanged(videoEnabled: Boolean) {
+ mRenderer.disableVideo(!videoEnabled)
+ }
+
+ override fun getView(): View {
+ return mView
+ }
+
+ override fun onPause() {
+ mView.onPause()
+ }
+
+ override fun onResume() {
+ mView.onResume()
+ }
+
+ companion object {
+ private const val THUMBNAIL_SIZE = 90 //in dp
+ }
+
+ init {
+ mView.setEGLContextClientVersion(2)
+ mView.setEGLConfigChooser(8, 8, 8, 8, 16, 0)
+ mView.holder.setFormat(PixelFormat.TRANSLUCENT)
+ mRenderer = MyRenderer()
+ mView.setRenderer(mRenderer)
+ mView.renderMode = GLSurfaceView.RENDERMODE_WHEN_DIRTY
+ }
+}
\ No newline at end of file
diff --git a/android/app/src/main/kotlin/com/hmg/hmgDr/util/ThumbnailCircleVideoRenderer.kt b/android/app/src/main/kotlin/com/hmg/hmgDr/util/ThumbnailCircleVideoRenderer.kt
new file mode 100644
index 00000000..b9b5a245
--- /dev/null
+++ b/android/app/src/main/kotlin/com/hmg/hmgDr/util/ThumbnailCircleVideoRenderer.kt
@@ -0,0 +1,357 @@
+package com.hmg.hmgDr.util
+
+import android.content.Context
+import android.content.res.Resources
+import android.graphics.PixelFormat
+import android.opengl.GLES20
+import android.opengl.GLSurfaceView
+import android.opengl.Matrix
+import android.view.View
+import com.opentok.android.BaseVideoRenderer
+import java.nio.ByteBuffer
+import java.nio.ByteOrder
+import java.nio.FloatBuffer
+import java.nio.ShortBuffer
+import java.util.concurrent.locks.ReentrantLock
+import javax.microedition.khronos.egl.EGLConfig
+import javax.microedition.khronos.opengles.GL10
+
+
+class ThumbnailCircleVideoRenderer(private val mContext: Context) : BaseVideoRenderer() {
+ private val mView: GLSurfaceView = GLSurfaceView(mContext)
+ private val mRenderer: MyRenderer
+
+ interface ThumbnailCircleVideoRendererMetadataListener {
+ fun onMetadataReady(metadata: ByteArray?)
+ }
+
+ fun setThumbnailCircleVideoRendererMetadataListener(metadataListener: ThumbnailCircleVideoRendererMetadataListener?) {
+ mRenderer.metadataListener = metadataListener
+ }
+
+ internal class MyRenderer : GLSurfaceView.Renderer {
+ var mTextureIds = IntArray(3)
+ var mScaleMatrix = FloatArray(16)
+ private val mVertexBuffer: FloatBuffer
+ private val mTextureBuffer: FloatBuffer
+ private val mDrawListBuffer: ShortBuffer
+ var mVideoFitEnabled = true
+ var mVideoDisabled = false
+ private val mVertexIndex = shortArrayOf(0, 1, 2, 0, 2, 3) // order to draw
+
+ // vertices
+ private val vertexShaderCode = """uniform mat4 uMVPMatrix;attribute vec4 aPosition;
+attribute vec2 aTextureCoord;
+varying vec2 vTextureCoord;
+void main() {
+ gl_Position = uMVPMatrix * aPosition;
+ vTextureCoord = aTextureCoord;
+}
+"""
+ private val fragmentShaderCode = """precision mediump float;
+uniform sampler2D Ytex;
+uniform sampler2D Utex,Vtex;
+uniform vec2 radiusDp;
+varying vec2 vTextureCoord;
+void main(void) {
+ float nx,ny,r,g,b,y,u,v;
+ mediump vec4 txl,ux,vx; nx=vTextureCoord[0];
+ ny=vTextureCoord[1];
+ y=texture2D(Ytex,vec2(nx,ny)).r;
+ u=texture2D(Utex,vec2(nx,ny)).r;
+ v=texture2D(Vtex,vec2(nx,ny)).r;
+ y=1.1643*(y-0.0625);
+ u=u-0.5;
+ v=v-0.5;
+ r=y+1.5958*v;
+ g=y-0.39173*u-0.81290*v;
+ b=y+2.017*u;
+ float radius = 0.5;
+ vec4 color0 = vec4(0.0, 0.0, 0.0, 0.0);
+ vec4 color1 = vec4(r, g, b, 1.0);
+ vec2 st = (gl_FragCoord.xy/radiusDp.xy); float dist = radius - distance(st,vec2(0.5));
+ float t = 1.0;
+ if (dist < 0.0) t = 0.0;
+ gl_FragColor = mix(color0, color1, t);
+}
+"""
+ var mFrameLock = ReentrantLock()
+ var mCurrentFrame: Frame? = null
+ private var mProgram = 0
+ private var mTextureWidth = 0
+ private var mTextureHeight = 0
+ private var mViewportWidth = 0
+ private var mViewportHeight = 0
+ override fun onSurfaceCreated(gl: GL10, config: EGLConfig) {
+ gl.glClearColor(0f, 0f, 0f, 1f)
+ GLES20.glClear(GLES20.GL_COLOR_BUFFER_BIT)
+ val vertexShader = loadShader(GLES20.GL_VERTEX_SHADER,
+ vertexShaderCode)
+ val fragmentShader = loadShader(GLES20.GL_FRAGMENT_SHADER,
+ fragmentShaderCode)
+ mProgram = GLES20.glCreateProgram() // create empty OpenGL ES
+ // Program
+ GLES20.glAttachShader(mProgram, vertexShader) // add the vertex
+ // shader to program
+ GLES20.glAttachShader(mProgram, fragmentShader) // add the fragment
+ // shader to
+ // program
+ GLES20.glLinkProgram(mProgram)
+ val positionHandle = GLES20.glGetAttribLocation(mProgram,
+ "aPosition")
+ val textureHandle = GLES20.glGetAttribLocation(mProgram,
+ "aTextureCoord")
+ GLES20.glVertexAttribPointer(positionHandle, COORDS_PER_VERTEX,
+ GLES20.GL_FLOAT, false, COORDS_PER_VERTEX * 4,
+ mVertexBuffer)
+ GLES20.glEnableVertexAttribArray(positionHandle)
+ GLES20.glVertexAttribPointer(textureHandle,
+ TEXTURECOORDS_PER_VERTEX, GLES20.GL_FLOAT, false,
+ TEXTURECOORDS_PER_VERTEX * 4, mTextureBuffer)
+ GLES20.glEnableVertexAttribArray(textureHandle)
+ GLES20.glUseProgram(mProgram)
+ var i = GLES20.glGetUniformLocation(mProgram, "Ytex")
+ GLES20.glUniform1i(i, 0) /* Bind Ytex to texture unit 0 */
+ i = GLES20.glGetUniformLocation(mProgram, "Utex")
+ GLES20.glUniform1i(i, 1) /* Bind Utex to texture unit 1 */
+ i = GLES20.glGetUniformLocation(mProgram, "Vtex")
+ GLES20.glUniform1i(i, 2) /* Bind Vtex to texture unit 2 */
+ val radiusDpLocation = GLES20.glGetUniformLocation(mProgram, "radiusDp")
+ val radiusDp = (Resources.getSystem().displayMetrics.density * THUMBNAIL_SIZE).toInt()
+ GLES20.glUniform2f(radiusDpLocation, radiusDp.toFloat(), radiusDp.toFloat())
+ mTextureWidth = 0
+ mTextureHeight = 0
+ }
+
+ fun setupTextures(frame: Frame) {
+ if (mTextureIds[0] != 0) {
+ GLES20.glDeleteTextures(3, mTextureIds, 0)
+ }
+ GLES20.glGenTextures(3, mTextureIds, 0)
+ val w = frame.width
+ val h = frame.height
+ val hw = w + 1 shr 1
+ val hh = h + 1 shr 1
+ initializeTexture(GLES20.GL_TEXTURE0, mTextureIds[0], w, h)
+ initializeTexture(GLES20.GL_TEXTURE1, mTextureIds[1], hw, hh)
+ initializeTexture(GLES20.GL_TEXTURE2, mTextureIds[2], hw, hh)
+ mTextureWidth = frame.width
+ mTextureHeight = frame.height
+ }
+
+ fun updateTextures(frame: Frame) {
+ val width = frame.width
+ val height = frame.height
+ val half_width = width + 1 shr 1
+ val half_height = height + 1 shr 1
+ val y_size = width * height
+ val uv_size = half_width * half_height
+ val bb = frame.buffer
+ // If we are reusing this frame, make sure we reset position and
+ // limit
+ bb.clear()
+ if (bb.remaining() == y_size + uv_size * 2) {
+ bb.position(0)
+ GLES20.glPixelStorei(GLES20.GL_UNPACK_ALIGNMENT, 1)
+ GLES20.glPixelStorei(GLES20.GL_PACK_ALIGNMENT, 1)
+ GLES20.glActiveTexture(GLES20.GL_TEXTURE0)
+ GLES20.glBindTexture(GLES20.GL_TEXTURE_2D, mTextureIds[0])
+ GLES20.glTexSubImage2D(GLES20.GL_TEXTURE_2D, 0, 0, 0, width,
+ height, GLES20.GL_LUMINANCE, GLES20.GL_UNSIGNED_BYTE,
+ bb)
+ bb.position(y_size)
+ GLES20.glActiveTexture(GLES20.GL_TEXTURE1)
+ GLES20.glBindTexture(GLES20.GL_TEXTURE_2D, mTextureIds[1])
+ GLES20.glTexSubImage2D(GLES20.GL_TEXTURE_2D, 0, 0, 0,
+ half_width, half_height, GLES20.GL_LUMINANCE,
+ GLES20.GL_UNSIGNED_BYTE, bb)
+ bb.position(y_size + uv_size)
+ GLES20.glActiveTexture(GLES20.GL_TEXTURE2)
+ GLES20.glBindTexture(GLES20.GL_TEXTURE_2D, mTextureIds[2])
+ GLES20.glTexSubImage2D(GLES20.GL_TEXTURE_2D, 0, 0, 0,
+ half_width, half_height, GLES20.GL_LUMINANCE,
+ GLES20.GL_UNSIGNED_BYTE, bb)
+ } else {
+ mTextureWidth = 0
+ mTextureHeight = 0
+ }
+ }
+
+ override fun onSurfaceChanged(gl: GL10, width: Int, height: Int) {
+ GLES20.glViewport(0, 0, width, height)
+ mViewportWidth = width
+ mViewportHeight = height
+ }
+
+ var metadataListener: ThumbnailCircleVideoRendererMetadataListener? = null
+ override fun onDrawFrame(gl: GL10) {
+ gl.glClearColor(0f, 0f, 0f, 0f)
+ GLES20.glClear(GLES20.GL_COLOR_BUFFER_BIT)
+ mFrameLock.lock()
+ if (mCurrentFrame != null && !mVideoDisabled) {
+ GLES20.glUseProgram(mProgram)
+ if (mTextureWidth != mCurrentFrame!!.width
+ || mTextureHeight != mCurrentFrame!!.height) {
+ setupTextures(mCurrentFrame!!)
+ }
+ updateTextures(mCurrentFrame!!)
+ Matrix.setIdentityM(mScaleMatrix, 0)
+ var scaleX = 1.0f
+ var scaleY = 1.0f
+ val ratio = (mCurrentFrame!!.width.toFloat()
+ / mCurrentFrame!!.height)
+ val vratio = mViewportWidth.toFloat() / mViewportHeight
+ if (mVideoFitEnabled) {
+ if (ratio > vratio) {
+ scaleY = vratio / ratio
+ } else {
+ scaleX = ratio / vratio
+ }
+ } else {
+ if (ratio < vratio) {
+ scaleY = vratio / ratio
+ } else {
+ scaleX = ratio / vratio
+ }
+ }
+ Matrix.scaleM(mScaleMatrix, 0,
+ scaleX * if (mCurrentFrame!!.isMirroredX) -1.0f else 1.0f,
+ scaleY, 1f)
+ metadataListener?.onMetadataReady(mCurrentFrame!!.metadata)
+ val mMVPMatrixHandle = GLES20.glGetUniformLocation(mProgram,
+ "uMVPMatrix")
+ GLES20.glUniformMatrix4fv(mMVPMatrixHandle, 1, false,
+ mScaleMatrix, 0)
+ GLES20.glDrawElements(GLES20.GL_TRIANGLES, mVertexIndex.size,
+ GLES20.GL_UNSIGNED_SHORT, mDrawListBuffer)
+ } else {
+ //black frame when video is disabled
+ gl.glClearColor(0f, 0f, 0f, 1f)
+ GLES20.glClear(GLES20.GL_COLOR_BUFFER_BIT)
+ }
+ mFrameLock.unlock()
+ }
+
+ fun displayFrame(frame: Frame?) {
+ mFrameLock.lock()
+ if (mCurrentFrame != null) {
+ mCurrentFrame!!.recycle()
+ }
+ mCurrentFrame = frame
+ mFrameLock.unlock()
+ }
+
+ fun disableVideo(b: Boolean) {
+ mFrameLock.lock()
+ mVideoDisabled = b
+ if (mVideoDisabled) {
+ if (mCurrentFrame != null) {
+ mCurrentFrame!!.recycle()
+ }
+ mCurrentFrame = null
+ }
+ mFrameLock.unlock()
+ }
+
+ fun enableVideoFit(enableVideoFit: Boolean) {
+ mVideoFitEnabled = enableVideoFit
+ }
+
+ companion object {
+ // number of coordinates per vertex in this array
+ const val COORDS_PER_VERTEX = 3
+ const val TEXTURECOORDS_PER_VERTEX = 2
+ var mXYZCoords = floatArrayOf(
+ -1.0f, 1.0f, 0.0f, // top left
+ -1.0f, -1.0f, 0.0f, // bottom left
+ 1.0f, -1.0f, 0.0f, // bottom right
+ 1.0f, 1.0f, 0.0f // top right
+ )
+ var mUVCoords = floatArrayOf(0f, 0f, 0f, 1f, 1f, 1f, 1f, 0f)
+ fun initializeTexture(name: Int, id: Int, width: Int, height: Int) {
+ GLES20.glActiveTexture(name)
+ GLES20.glBindTexture(GLES20.GL_TEXTURE_2D, id)
+ GLES20.glTexParameterf(GLES20.GL_TEXTURE_2D,
+ GLES20.GL_TEXTURE_MIN_FILTER, GLES20.GL_NEAREST.toFloat())
+ GLES20.glTexParameterf(GLES20.GL_TEXTURE_2D,
+ GLES20.GL_TEXTURE_MAG_FILTER, GLES20.GL_LINEAR.toFloat())
+ GLES20.glTexParameterf(GLES20.GL_TEXTURE_2D,
+ GLES20.GL_TEXTURE_WRAP_S, GLES20.GL_CLAMP_TO_EDGE.toFloat())
+ GLES20.glTexParameterf(GLES20.GL_TEXTURE_2D,
+ GLES20.GL_TEXTURE_WRAP_T, GLES20.GL_CLAMP_TO_EDGE.toFloat())
+ GLES20.glTexImage2D(GLES20.GL_TEXTURE_2D, 0, GLES20.GL_LUMINANCE,
+ width, height, 0, GLES20.GL_LUMINANCE,
+ GLES20.GL_UNSIGNED_BYTE, null)
+ }
+
+ fun loadShader(type: Int, shaderCode: String?): Int {
+ val shader = GLES20.glCreateShader(type)
+ GLES20.glShaderSource(shader, shaderCode)
+ GLES20.glCompileShader(shader)
+ return shader
+ }
+ }
+
+ init {
+ val bb = ByteBuffer.allocateDirect(mXYZCoords.size * 4)
+ bb.order(ByteOrder.nativeOrder())
+ mVertexBuffer = bb.asFloatBuffer()
+ mVertexBuffer.put(mXYZCoords)
+ mVertexBuffer.position(0)
+ val tb = ByteBuffer.allocateDirect(mUVCoords.size * 4)
+ tb.order(ByteOrder.nativeOrder())
+ mTextureBuffer = tb.asFloatBuffer()
+ mTextureBuffer.put(mUVCoords)
+ mTextureBuffer.position(0)
+ val dlb = ByteBuffer.allocateDirect(mVertexIndex.size * 2)
+ dlb.order(ByteOrder.nativeOrder())
+ mDrawListBuffer = dlb.asShortBuffer()
+ mDrawListBuffer.put(mVertexIndex)
+ mDrawListBuffer.position(0)
+ }
+ }
+
+ override fun onFrame(frame: Frame) {
+ mRenderer.displayFrame(frame)
+ mView.requestRender()
+ }
+
+ override fun setStyle(key: String, value: String) {
+ if (STYLE_VIDEO_SCALE == key) {
+ if (STYLE_VIDEO_FIT == value) {
+ mRenderer.enableVideoFit(true)
+ } else if (STYLE_VIDEO_FILL == value) {
+ mRenderer.enableVideoFit(false)
+ }
+ }
+ }
+
+ override fun onVideoPropertiesChanged(videoEnabled: Boolean) {
+ mRenderer.disableVideo(!videoEnabled)
+ }
+
+ override fun getView(): View {
+ return mView
+ }
+
+ override fun onPause() {
+ mView.onPause()
+ }
+
+ override fun onResume() {
+ mView.onResume()
+ }
+
+ companion object {
+ private const val THUMBNAIL_SIZE = 90 //in dp
+ }
+
+ init {
+ mView.setEGLContextClientVersion(2)
+ mView.setEGLConfigChooser(8, 8, 8, 8, 16, 0)
+ mView.holder.setFormat(PixelFormat.TRANSLUCENT)
+ mRenderer = MyRenderer()
+ mView.setRenderer(mRenderer)
+ mView.renderMode = GLSurfaceView.RENDERMODE_WHEN_DIRTY
+ }
+}
diff --git a/android/app/src/main/res/layout/activity_video_call.xml b/android/app/src/main/res/layout/activity_video_call.xml
index a7470da3..a54e57b3 100644
--- a/android/app/src/main/res/layout/activity_video_call.xml
+++ b/android/app/src/main/res/layout/activity_video_call.xml
@@ -97,8 +97,7 @@
android:layout_alignParentTop="true"
android:layout_alignParentEnd="true"
android:layout_marginTop="@dimen/local_preview_margin_top"
- android:layout_marginEnd="@dimen/local_preview_margin_top"
- android:background="@color/localBackground">
+ android:layout_marginEnd="@dimen/local_preview_margin_top">
+
+