Android Controller

From Immersive Visualization Lab Wiki
Revision as of 21:12, 4 June 2011 by Jeanne (Talk | contribs)

Jump to: navigation, search

Contents

Android Controller

Camera.jpg Jeanne Wang

Objective

Create an intuitive and novel approach to a tablet based multi-touch and sensor-enabled controller for a real-time 3D visualization on a 2D platform. Using camera pose information relative to a 3D model displayed on a screen, we can display virtual camera shots in the 3D model space on the tablet.

Android Basics:

  • Sandboxed in a linux environment, each application is actually a user
  • Model View Controller setup
    • Model - Content providers
    • View - XML
    • Controller - Activity, or Service
  • UI Views

MainUI.png 

<?xml version="1.0" encoding="utf-8"?>
<LinearLayout xmlns:android="http://schemas.android.com/apk/res/android"
	android:layout_width="match_parent"
    android:layout_height="match_parent"
    android:orientation="vertical" >
    <LinearLayout
    	android:layout_width="wrap_content"
    	android:layout_height="wrap_content"
    	android:padding="10dp">
        <SeekBar android:id="@+id/seekBar1" android:layout_marginLeft="10dip" android:layout_width="200dip" android:layout_height="wrap_content"></SeekBar>
	    <TextView android:text="@string/hello" 
			android:layout_height="wrap_content" 
			android:layout_marginLeft="10dip" 
			android:id="@+id/seekBarText" 
			android:layout_width="wrap_content" 
			android:layout_alignParentRight="true">
			</TextView>
	</LinearLayout>
	<RelativeLayout
		android:layout_width="fill_parent"
	    android:layout_height="fill_parent">
		<Button 
			android:layout_height="wrap_content" 
			android:id="@+id/button2" 
			android:layout_width="wrap_content" 
			android:text="type some text" 
			android:layout_below="@+id/button3" 
			android:layout_alignRight="@+id/button3">
		</Button>
		<Button 
			android:layout_height="wrap_content" 
			android:id="@+id/button4" 
			android:layout_width="wrap_content" 
			android:text="camera" 
			android:layout_below="@+id/button2">
		</Button>
		<Button 
		android:layout_height="wrap_content" 
		android:id="@+id/button1" 
		android:layout_width="wrap_content" 
		android:text="socket send" 
		android:layout_below="@+id/button4" 
		android:layout_alignLeft="@+id/button4" 
		android:layout_alignRight="@+id/button4">
		</Button>
		<Button 
		android:layout_height="wrap_content" 
		android:id="@+id/button3" 
		android:layout_width="wrap_content" 
		android:text="touch screen" 
		android:layout_alignParentLeft="true"></Button>
	</RelativeLayout>
	<ImageView
		android:id="@+id/photoResultView"
		android:layout_width="wrap_content" 
		android:layout_height="wrap_content" 
	></ImageView>
</LinearLayout>
  • UI Control Component
    • Buttons 
Button button = (Button)findViewById(R.id.button1);
button.setOnClickListener(buttonOnClick);
    • Sliders 
SeekBar seekbar=(SeekBar)findViewById(R.id.seekBar1);
seekbar.setOnSeekBarChangeListener(sbar);
    • TextView 
TextView seekbartxt=(TextView)findViewById(R.id.seekBarText);
			CharSequence t="slider progress:"+progress;
			seekbartxt.setText(t);
    • EditText
      • In Android manifest make sure to enable soft keyboard 
<activity android:name=".TypeText"
        		  android:windowSoftInputMode="stateAlwaysVisible|adjustResize">
        </activity>

EditText edtView=(EditText)findViewById(R.id.EditText01);
edtView.setInputType(InputType.TYPE_CLASS_TEXT);
    • Toasts (timed popups) 
                    int duration = Toast.LENGTH_SHORT;
                    View v;
		    CharSequence text = "button "+v.getId()+" pressed";
		    Toast t=Toast.makeText(v.getContext(),text, duration);
		    t.show();
  • Listeners
    • OnClickListener 
private OnClickListener showCamera = new OnClickListener() {
	    public void onClick(View v) {
		    	//do stuff here
            }
    • OnSeekBarChangeListener 
private SeekBar.OnSeekBarChangeListener sbar = new SeekBar.OnSeekBarChangeListener()
	{
		
		public void onStopTrackingTouch(SeekBar seekBar) {			
		}
		
		public void onStartTrackingTouch(SeekBar seekBar) {
		}
		
		public void onProgressChanged(SeekBar seekBar, int progress,
				boolean fromUser) {
		}
	};
    • OnTouchListener 
LinearLayout screen =(LinearLayout)findViewById(R.id.touch);
screen.setOnTouchListener(new View.OnTouchListener() {
            public boolean onTouch(View v, MotionEvent e) {
            	float x = e.getX();
            	float y = e.getY();
    			TextView text=(TextView)findViewById(R.id.touchtext);
    			CharSequence t="x:"+x+", y:"+y;
    			text.setText(t);
				return true;
         
            }
        });
  • Start New Intent 
Intent myIntent = new Intent(v.getContext(), TouchScreen.class);
		    startActivityForResult(myIntent, 0);
  • Sockets
    • In Android Manifest be sure to enable internet usage 
<uses-permission android:name="android.permission.INTERNET" />

public void sendSocket(String sendStr, String textStr)
	   {
	       try {
	           //Socket s = new Socket("137.110.119.121",11011);   // TourCAVE
	    	   Socket s = new Socket("137.110.118.26",3412); // sessions
	    	   //Socket s = new Socket("137.110.115.194",11011);   // HP laptop

	           //outgoing stream redirect to socket
	           OutputStream out = s.getOutputStream();
	           

	           PrintWriter output = new PrintWriter(out);
	           output.println(sendStr);
	           output.close();  // required to actually send the text
	           
	           Context context = getApplicationContext();
	           Toast.makeText(context, textStr, Toast.LENGTH_SHORT).show();

	           //Close connection
	           s.close();
	       } catch (UnknownHostException e) {
	    	   Context context = getApplicationContext();
	           Toast.makeText(context, "UnknownHostException!",
	        		   Toast.LENGTH_SHORT).show();
	           e.printStackTrace();
	       } catch (IOException e) {
	    	   Context context = getApplicationContext();
	           Toast.makeText(context, "IOException! "+e,
	        		   Toast.LENGTH_SHORT).show();
	           e.printStackTrace();
	       }
	   }
    • Using sockets to get IP Address 
Socket s = new Socket("137.110.115.194",11011);   // HP laptop
String myIPAddress = s.getLocalAddress().toString();
  • Camera
    • In Android manifest be sure to include camera capabilities 
<uses-permission android:name="android.permission.CAMERA" />
<uses-feature android:name="android.hardware.camera" />
<uses-feature android:name="android.hardware.camera.autofocus" />
    • Default Camera 
                private static int CAMERA_PIC_REQUEST = 10232;
	    	Intent intent = new Intent(MediaStore.ACTION_IMAGE_CAPTURE);
	    	intent.putExtra(MediaStore.EXTRA_VIDEO_QUALITY, 1);
	    	startActivityForResult(intent, CAMERA_PIC_REQUEST);

	protected void onActivityResult(int requestCode, int resultCode, Intent data) {  
	    if (requestCode == CAMERA_PIC_REQUEST) {  
	    	android.graphics.Bitmap thumbnail = (android.graphics.Bitmap) data.getExtras().get("data");    
	    	ImageView image = (ImageView) findViewById(R.id.photoResultView);  
	    	image.setImageBitmap(thumbnail); 
	    }  
	}
    • Custom Camera 

package jeanne.prime.tablet;


import java.io.FileNotFoundException;
import java.io.FileOutputStream;
import java.io.IOException;

import android.content.Context;
import android.graphics.Canvas;
import android.graphics.Color;
import android.graphics.Paint;
import android.hardware.Camera;
import android.hardware.Camera.PreviewCallback;
import android.util.Log;
import android.view.SurfaceHolder;
import android.view.SurfaceView;

class Preview extends SurfaceView implements SurfaceHolder.Callback {
	private static final String TAG = "Preview";

	SurfaceHolder mHolder;
	public Camera camera;

	Preview(Context context) {
		super(context);

		// Install a SurfaceHolder.Callback so we get notified when the
		// underlying surface is created and destroyed.
		mHolder = getHolder();
		mHolder.addCallback(this);
		mHolder.setType(SurfaceHolder.SURFACE_TYPE_PUSH_BUFFERS);
	}

	public void surfaceCreated(SurfaceHolder holder) {
		// The Surface has been created, acquire the camera and tell it where
		// to draw.
		camera = Camera.open();
		try {
			camera.setPreviewDisplay(holder);

			camera.setPreviewCallback(new PreviewCallback() {

				public void onPreviewFrame(byte[] data, Camera arg1) {
					//FileOutputStream outStream = null;
					try {
						Log.d(TAG, "onPreviewFrame - wrote bytes: "
								+ data.length);

					} finally {
					}
					Preview.this.invalidate();
				}
			});
		} catch (IOException e) {
			e.printStackTrace();
		}
	}

	public void surfaceDestroyed(SurfaceHolder holder) {
		// Surface will be destroyed when we return, so stop the preview.
		// Because the CameraDevice object is not a shared resource, it's very
		// important to release it when the activity is paused.
		camera.stopPreview();
		camera.release();
		camera = null;
		Log.d(TAG,"surfaceDestroyed");
	}

	public void surfaceChanged(SurfaceHolder holder, int format, int w, int h) {
		// Now that the size is known, set up the camera parameters and begin
		// the preview.
		Camera.Parameters parameters = camera.getParameters();
		parameters.setPreviewSize(w, h);
		parameters.setPictureSize(100, 60);
		camera.setParameters(parameters);
		camera.startPreview();
	}

	@Override
	public void draw(Canvas canvas) {
		super.draw(canvas);
		Paint p = new Paint(Color.RED);
		Log.d(TAG, "draw");
		canvas.drawText("PREVIEW", canvas.getWidth() / 2,
				canvas.getHeight() / 2, p);
	}
}

package jeanne.prime.tablet;

import java.io.BufferedOutputStream;
import java.io.ByteArrayOutputStream;
import java.io.FileNotFoundException;
import java.io.FileOutputStream;
import java.io.IOException;
import java.io.InputStream;
import java.io.OutputStream;
import java.io.PrintWriter;
import java.net.Socket;
import java.net.UnknownHostException;

import android.app.Activity;
import android.content.Context;
import android.content.Intent;
import android.hardware.Camera;
import android.hardware.Camera.PictureCallback;
import android.hardware.Camera.ShutterCallback;
import android.os.Bundle;
import android.util.Log;
import android.view.SurfaceHolder;
import android.view.View;
import android.view.View.OnClickListener;
import android.widget.Button;
import android.widget.FrameLayout;
import android.widget.Toast;

/*
 * http://marakana.com/forums/android/examples/39.html
 * http://developer.android.com/resources/samples/ApiDemos/src/com/example/android/apis/graphics/CameraPreview.html 
 * http://mobile.tutsplus.com/tutorials/android/android-sdk-quick-tip-launching-the-camera/
 */

public class CameraDemo extends Activity {
	private static final String TAG = "CameraDemo";
	Preview preview;
	Button buttonClick;
	Socket s;

	/** Called when the activity is first created. */
	@Override
	public void onCreate(Bundle savedInstanceState) {
		super.onCreate(savedInstanceState);
		setContentView(R.layout.camera);

		preview = new Preview(this);
		((FrameLayout) findViewById(R.id.preview)).addView(preview);
		
		buttonClick = (Button) findViewById(R.id.buttonClick);
		buttonClick.setOnClickListener(new OnClickListener() {
			public void onClick(View v) {
				//preview.camera.startPreview();
				preview.camera.takePicture(null, rawCallback,
						jpegCallback);

			}
		});
		
		Button back = (Button) findViewById(R.id.backbutton);
        back.setOnClickListener(new View.OnClickListener() {
            public void onClick(View view) {
                Intent intent = new Intent();
                setResult(Activity.RESULT_OK, intent);
                finish();
            }

        });

		Log.d(TAG, "onCreate'd");
	}
	public byte[] intToByteArray(int value) { //big endian
        return new byte[] {
                (byte)(value >>> 24),
                (byte)(value >>> 16),
                (byte)(value >>> 8),
                (byte)value};
	}
	public byte[] intToByteArray2(int value) { //little endian
        return new byte[] {
        		(byte)value,
                (byte)(value >>> 8),
                (byte)(value >>> 16),
                (byte)(value >>> 24)
                };
	}
	
	public void sendPhoto(byte[] photo, String textStr)
	   {
	       try {
	       
	    	   s = new Socket("137.110.118.26",3412);// sessions

	    	   s.setTcpNoDelay(true);
	           OutputStream out = s.getOutputStream();
	           //BufferedOutputStream out = new BufferedOutputStream(o);

	           out.write(intToByteArray2(photo.length), 0, 4);
	           out.close();
	           
	           Socket s2 = new Socket("137.110.118.26",3412);// sessions

	    	   s2.setTcpNoDelay(true);
	           OutputStream out2 = s2.getOutputStream();
	           out2.write(photo);
	           out2.close();
	           
	           Context context = getApplicationContext();
	           Toast.makeText(context, textStr, Toast.LENGTH_SHORT).show();
	           Log.d(TAG,textStr);

	       } catch (UnknownHostException e) {
	    	   Context context = getApplicationContext();
	           Toast.makeText(context, "UnknownHostException!",
	        		   Toast.LENGTH_SHORT).show();
	           Log.d(TAG,"UnknownHostException!");
	           e.printStackTrace();
	       } catch (IOException e) {
	    	   Context context = getApplicationContext();
	           Toast.makeText(context, "IOException! "+e,
	        		   Toast.LENGTH_SHORT).show();
	           Log.d(TAG,"IOException! "+e);
	           e.printStackTrace();
	       }
	   }
	
	ShutterCallback shutterCallback = new ShutterCallback() {
		public void onShutter() {
			Log.d(TAG, "onShutter'd");
		}
	};

	/** Handles data for raw picture */
	PictureCallback rawCallback = new PictureCallback() {
		public void onPictureTaken(byte[] data, Camera camera) {
			camera.startPreview();
			if (data != null){
				sendPhoto(data, "photosize:"+data.length);
			}
			Log.d(TAG, "onPictureTaken - raw");
		}
	};

	/** Handles data for jpeg picture */
	PictureCallback jpegCallback = new PictureCallback() {
		public void onPictureTaken(byte[] data, Camera camera) {
			if (data != null){
				sendPhoto(data, "photosize:"+data.length);
				Log.d(TAG, "onPictureTaken - jpeg,size:"+data.length);
			}
			
		}
	};
	
	public void onPause(){
        //Close connection
        try {
			s.close();
		} catch (IOException e) {
			// TODO Auto-generated catch block
			e.printStackTrace();
		}
    }

}

Hardware + Emulator

Network

  • server
  • client
  • socket issues

ARToolkit

  • detect marker
  • pose

Android

OpenCover

  • display marker
  • display dataset

Tips

  • Tip: An easy way to add import packages to your project is to press Ctrl-Shift-O (Cmd-Shift-O, on Mac). This is an Eclipse shortcut that identifies missing packages based on your code and adds them for you.
  • To get cpuinfo on a particular machine look in: /proc/cpuinfo
  • To get ipaddress on a particular machine call /sbin/ifconfig

Other possibilities

  • Corona Sdk [[1]]
  • HTML5 DeviceMotionEvent [[2]]
  • PhoneGap [[3]]
Retrieved from "http://ivl.calit2.net/wiki/index.php?title=Android_Controller&oldid=3761"