fixed classifier and added in a preliminary results view that shows what pokemon are currently being looked at.
This commit is contained in:
+50
-73
@@ -1,42 +1,35 @@
|
|||||||
import 'dart:math';
|
|
||||||
import 'dart:ui';
|
|
||||||
|
|
||||||
import 'package:collection/collection.dart';
|
import 'package:collection/collection.dart';
|
||||||
import 'package:image/image.dart' as image_lib;
|
import 'package:image/image.dart' as image_lib;
|
||||||
import 'package:tflite_flutter/tflite_flutter.dart';
|
import 'package:tflite_flutter/tflite_flutter.dart';
|
||||||
import 'package:tflite_flutter_helper/tflite_flutter_helper.dart';
|
import 'package:tflite_flutter_helper/tflite_flutter_helper.dart';
|
||||||
|
|
||||||
import '../utils/logger.dart';
|
import '../utils/logger.dart';
|
||||||
import '../utils/recognition.dart';
|
import 'data/recognition.dart';
|
||||||
import '../utils/stats.dart';
|
import 'data/stats.dart';
|
||||||
|
|
||||||
/// Classifier
|
/// Classifier
|
||||||
class Classifier {
|
class Classifier {
|
||||||
static const String MODEL_FILE_NAME = "detect.tflite";
|
static const String modelFileName = "efficientnet_v2s.tflite";
|
||||||
static const String LABEL_FILE_NAME = "labelmap.txt";
|
static const int inputSize = 224;
|
||||||
|
|
||||||
/// Input size of image (height = width = 300)
|
|
||||||
static const int INPUT_SIZE = 224;
|
|
||||||
|
|
||||||
/// Result score threshold
|
|
||||||
static const double THRESHOLD = 0.5;
|
|
||||||
|
|
||||||
/// [ImageProcessor] used to pre-process the image
|
/// [ImageProcessor] used to pre-process the image
|
||||||
ImageProcessor? imageProcessor;
|
ImageProcessor? imageProcessor;
|
||||||
|
|
||||||
/// Padding the image to transform into square
|
///Tensor image to move image data into
|
||||||
// int padSize = 0;
|
late TensorImage _inputImage;
|
||||||
|
|
||||||
/// Instance of Interpreter
|
/// Instance of Interpreter
|
||||||
late Interpreter _interpreter;
|
late Interpreter _interpreter;
|
||||||
|
|
||||||
late TensorBuffer _outputBuffer;
|
late TensorBuffer _outputBuffer;
|
||||||
late var _probabilityProcessor;
|
late TfLiteType _inputType;
|
||||||
|
late TfLiteType _outputType;
|
||||||
|
|
||||||
|
late SequentialProcessor<TensorBuffer> _outputProcessor;
|
||||||
|
|
||||||
/// Labels file loaded as list
|
/// Labels file loaded as list
|
||||||
late List<String> _labels;
|
late List<String> _labels;
|
||||||
|
int classifierCreationStart = -1;
|
||||||
|
|
||||||
/// Number of results to show
|
|
||||||
static const int NUM_RESULTS = 10;
|
|
||||||
|
|
||||||
Classifier({
|
Classifier({
|
||||||
Interpreter? interpreter,
|
Interpreter? interpreter,
|
||||||
@@ -51,19 +44,18 @@ class Classifier {
|
|||||||
try {
|
try {
|
||||||
_interpreter = interpreter ??
|
_interpreter = interpreter ??
|
||||||
await Interpreter.fromAsset(
|
await Interpreter.fromAsset(
|
||||||
MODEL_FILE_NAME,
|
modelFileName,
|
||||||
options: InterpreterOptions()..threads = 4,
|
options: InterpreterOptions()..threads = 8,
|
||||||
);
|
);
|
||||||
var outputTensor = _interpreter.getOutputTensor(0);
|
var outputTensor = _interpreter.getOutputTensor(0);
|
||||||
var outputShape = outputTensor.shape;
|
var outputShape = outputTensor.shape;
|
||||||
var outputType = outputTensor.type;
|
_outputType = outputTensor.type;
|
||||||
|
|
||||||
var inputTensor = _interpreter.getInputTensor(0);
|
var inputTensor = _interpreter.getInputTensor(0);
|
||||||
var intputShape = inputTensor.shape;
|
// var intputShape = inputTensor.shape;
|
||||||
var intputType = inputTensor.type;
|
_inputType = inputTensor.type;
|
||||||
|
_inputImage = TensorImage(_inputType);
|
||||||
_outputBuffer = TensorBuffer.createFixedSize(outputShape, outputType);
|
_outputBuffer = TensorBuffer.createFixedSize(outputShape, _outputType);
|
||||||
_probabilityProcessor =
|
_outputProcessor =
|
||||||
TensorProcessorBuilder().add(NormalizeOp(0, 1)).build();
|
TensorProcessorBuilder().add(NormalizeOp(0, 1)).build();
|
||||||
} catch (e) {
|
} catch (e) {
|
||||||
logger.e("Error while creating interpreter: ", e);
|
logger.e("Error while creating interpreter: ", e);
|
||||||
@@ -80,61 +72,45 @@ class Classifier {
|
|||||||
}
|
}
|
||||||
|
|
||||||
/// Pre-process the image
|
/// Pre-process the image
|
||||||
TensorImage? getProcessedImage(TensorImage inputImage) {
|
TensorImage? getProcessedImage(TensorImage? inputImage) {
|
||||||
// padSize = max(inputImage.height, inputImage.width);
|
// padSize = max(inputImage.height, inputImage.width);
|
||||||
|
if (inputImage != null) {
|
||||||
imageProcessor ??= ImageProcessorBuilder()
|
imageProcessor ??= ImageProcessorBuilder()
|
||||||
// .add(ResizeWithCropOrPadOp(padSize, padSize))
|
.add(ResizeWithCropOrPadOp(224, 224))
|
||||||
.add(ResizeOp(INPUT_SIZE, INPUT_SIZE, ResizeMethod.BILINEAR))
|
.add(ResizeOp(inputSize, inputSize, ResizeMethod.BILINEAR))
|
||||||
.add(NormalizeOp(127.5, 127.5))
|
// .add(NormalizeOp(127.5, 127.5))
|
||||||
.build();
|
.build();
|
||||||
return imageProcessor?.process(inputImage);
|
return imageProcessor?.process(inputImage);
|
||||||
}
|
}
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
|
||||||
/// Runs object detection on the input image
|
/// Runs object detection on the input image
|
||||||
Map<String, dynamic>? predict(image_lib.Image image) {
|
Map<String, dynamic>? predict(image_lib.Image image) {
|
||||||
logger.i(labels);
|
var preProcStart = DateTime.now().millisecondsSinceEpoch;
|
||||||
var predictStartTime = DateTime.now().millisecondsSinceEpoch;
|
_inputImage.loadImage(image);
|
||||||
if (_interpreter == null) {
|
_inputImage = getProcessedImage(_inputImage)!;
|
||||||
logger.e("Interpreter not initialized");
|
var inferenceStart = DateTime.now().millisecondsSinceEpoch;
|
||||||
return null;
|
_interpreter.run(_inputImage.buffer, _outputBuffer.getBuffer());
|
||||||
}
|
var postProcStart = DateTime.now().millisecondsSinceEpoch;
|
||||||
var preProcessStart = DateTime.now().millisecondsSinceEpoch;
|
|
||||||
// Create TensorImage from image
|
|
||||||
// Pre-process TensorImage
|
|
||||||
var procImage = getProcessedImage(TensorImage.fromImage(image));
|
|
||||||
|
|
||||||
var preProcessElapsedTime =
|
|
||||||
DateTime.now().millisecondsSinceEpoch - preProcessStart;
|
|
||||||
if (procImage != null) {
|
|
||||||
var inferenceTimeStart = DateTime.now().millisecondsSinceEpoch;
|
|
||||||
// run inference
|
|
||||||
var inferenceTimeElapsed =
|
|
||||||
DateTime.now().millisecondsSinceEpoch - inferenceTimeStart;
|
|
||||||
|
|
||||||
logger.i("Sending image to ML");
|
|
||||||
|
|
||||||
logger.i(procImage.buffer.asFloat32List());
|
|
||||||
logger.i(procImage.width);
|
|
||||||
logger.i(procImage.height);
|
|
||||||
logger.i(procImage.tensorBuffer.shape);
|
|
||||||
logger.i(procImage.tensorBuffer.isDynamic);
|
|
||||||
_interpreter.run(procImage.buffer, _outputBuffer.getBuffer());
|
|
||||||
|
|
||||||
Map<String, double> labeledProb = TensorLabel.fromList(
|
Map<String, double> labeledProb = TensorLabel.fromList(
|
||||||
labels, _probabilityProcessor.process(_outputBuffer))
|
labels, _outputProcessor.process(_outputBuffer))
|
||||||
.getMapWithFloatValue();
|
.getMapWithFloatValue();
|
||||||
final pred = getTopProbability(labeledProb);
|
final predictions = getTopProbabilities(labeledProb, number: 5)
|
||||||
Recognition rec = Recognition(1, pred.key, pred.value);
|
.mapIndexed(
|
||||||
var predictElapsedTime = DateTime.now().millisecondsSinceEpoch - predictStartTime;
|
(index, element) => Recognition(index, element.key, element.value))
|
||||||
|
.toList();
|
||||||
|
var endTime = DateTime.now().millisecondsSinceEpoch;
|
||||||
return {
|
return {
|
||||||
"recognitions": rec,
|
"recognitions": predictions,
|
||||||
"stats": Stats(predictElapsedTime, predictElapsedTime, predictElapsedTime, predictElapsedTime),
|
"stats": Stats(
|
||||||
|
totalTime: endTime - preProcStart,
|
||||||
|
preProcessingTime: inferenceStart - preProcStart,
|
||||||
|
inferenceTime: postProcStart - inferenceStart,
|
||||||
|
postProcessingTime: endTime - postProcStart,
|
||||||
|
),
|
||||||
};
|
};
|
||||||
} else {
|
|
||||||
return null;
|
|
||||||
}
|
}
|
||||||
}
|
|
||||||
|
|
||||||
/// Gets the interpreter instance
|
/// Gets the interpreter instance
|
||||||
Interpreter get interpreter => _interpreter;
|
Interpreter get interpreter => _interpreter;
|
||||||
|
|
||||||
@@ -142,11 +118,12 @@ class Classifier {
|
|||||||
List<String> get labels => _labels;
|
List<String> get labels => _labels;
|
||||||
}
|
}
|
||||||
|
|
||||||
MapEntry<String, double> getTopProbability(Map<String, double> labeledProb) {
|
List<MapEntry<String, double>> getTopProbabilities(
|
||||||
|
Map<String, double> labeledProb,
|
||||||
|
{int number = 3}) {
|
||||||
var pq = PriorityQueue<MapEntry<String, double>>(compare);
|
var pq = PriorityQueue<MapEntry<String, double>>(compare);
|
||||||
pq.addAll(labeledProb.entries);
|
pq.addAll(labeledProb.entries);
|
||||||
|
return [for (var i = 0; i < number; i += 1) pq.removeFirst()];
|
||||||
return pq.first;
|
|
||||||
}
|
}
|
||||||
|
|
||||||
int compare(MapEntry<String, double> e1, MapEntry<String, double> e2) {
|
int compare(MapEntry<String, double> e1, MapEntry<String, double> e2) {
|
||||||
|
|||||||
@@ -0,0 +1,18 @@
|
|||||||
|
|
||||||
|
class Stats {
|
||||||
|
int totalTime;
|
||||||
|
int preProcessingTime;
|
||||||
|
int inferenceTime;
|
||||||
|
int postProcessingTime;
|
||||||
|
|
||||||
|
Stats(
|
||||||
|
{this.totalTime = -1,
|
||||||
|
this.preProcessingTime = -1,
|
||||||
|
this.inferenceTime = -1,
|
||||||
|
this.postProcessingTime = -1});
|
||||||
|
|
||||||
|
@override
|
||||||
|
String toString() {
|
||||||
|
return 'Stats{totalPredictTime: $totalTime, preProcessingTime: $preProcessingTime, inferenceTime: $inferenceTime, postProcessingTime: $postProcessingTime}';
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,62 @@
|
|||||||
|
import 'dart:isolate';
|
||||||
|
|
||||||
|
import 'package:camera/camera.dart';
|
||||||
|
import 'package:tensordex_mobile/tflite/classifier.dart';
|
||||||
|
import 'package:tflite_flutter/tflite_flutter.dart';
|
||||||
|
|
||||||
|
import '../utils/image_utils.dart';
|
||||||
|
import '../utils/logger.dart';
|
||||||
|
|
||||||
|
class IsolateBase {
|
||||||
|
final ReceivePort _receivePort = ReceivePort();
|
||||||
|
}
|
||||||
|
|
||||||
|
class MLIsolate extends IsolateBase {
|
||||||
|
static const String debugIsolate = "MLIsolate";
|
||||||
|
late SendPort _sendPort;
|
||||||
|
|
||||||
|
SendPort get sendPort => _sendPort;
|
||||||
|
|
||||||
|
Future<void> start() async {
|
||||||
|
await Isolate.spawn<SendPort>(
|
||||||
|
entryPoint,
|
||||||
|
_receivePort.sendPort,
|
||||||
|
debugName: debugIsolate,
|
||||||
|
);
|
||||||
|
_sendPort = await _receivePort.first;
|
||||||
|
}
|
||||||
|
|
||||||
|
static void entryPoint(SendPort sendPort) async {
|
||||||
|
final port = ReceivePort();
|
||||||
|
sendPort.send(port.sendPort);
|
||||||
|
|
||||||
|
await for (final MLIsolateData mlIsolateData in port) {
|
||||||
|
var cameraImage = mlIsolateData.cameraImage;
|
||||||
|
var converted = ImageUtils.convertCameraImage(cameraImage);
|
||||||
|
if (converted != null) {
|
||||||
|
Classifier classifier = Classifier(
|
||||||
|
interpreter:
|
||||||
|
Interpreter.fromAddress(mlIsolateData.interpreterAddress),
|
||||||
|
labels: mlIsolateData.labels);
|
||||||
|
var result = classifier.predict(converted);
|
||||||
|
mlIsolateData.responsePort?.send(result);
|
||||||
|
} else {
|
||||||
|
mlIsolateData.responsePort?.send({"response": "not working yet"});
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Bundles data to pass between Isolate
|
||||||
|
class MLIsolateData {
|
||||||
|
CameraImage cameraImage;
|
||||||
|
int interpreterAddress;
|
||||||
|
List<String> labels;
|
||||||
|
SendPort? responsePort;
|
||||||
|
|
||||||
|
MLIsolateData(
|
||||||
|
this.cameraImage,
|
||||||
|
this.interpreterAddress,
|
||||||
|
this.labels,
|
||||||
|
);
|
||||||
|
}
|
||||||
+59
-123
@@ -2,16 +2,16 @@ import 'dart:isolate';
|
|||||||
|
|
||||||
import 'package:camera/camera.dart';
|
import 'package:camera/camera.dart';
|
||||||
import 'package:flutter/material.dart';
|
import 'package:flutter/material.dart';
|
||||||
import 'package:tensordex_mobile/tflite/classifier.dart';
|
import 'package:tensordex_mobile/tflite/ml_isolate.dart';
|
||||||
import 'package:tflite_flutter/tflite_flutter.dart';
|
import 'package:tflite_flutter/tflite_flutter.dart';
|
||||||
import 'package:tensordex_mobile/utils/image_utils.dart';
|
|
||||||
|
|
||||||
|
import '../tflite/classifier.dart';
|
||||||
import '../utils/logger.dart';
|
import '../utils/logger.dart';
|
||||||
import '../utils/recognition.dart';
|
import '../tflite/data/recognition.dart';
|
||||||
import '../utils/stats.dart';
|
import '../tflite/data/stats.dart';
|
||||||
|
|
||||||
/// [CameraView] sends each frame for inference
|
/// [PokedexView] sends each frame for inference
|
||||||
class CameraView extends StatefulWidget {
|
class PokedexView extends StatefulWidget {
|
||||||
/// Callback to pass results after inference to [HomeView]
|
/// Callback to pass results after inference to [HomeView]
|
||||||
final Function(List<Recognition> recognitions) resultsCallback;
|
final Function(List<Recognition> recognitions) resultsCallback;
|
||||||
|
|
||||||
@@ -19,32 +19,26 @@ class CameraView extends StatefulWidget {
|
|||||||
final Function(Stats stats) statsCallback;
|
final Function(Stats stats) statsCallback;
|
||||||
|
|
||||||
/// Constructor
|
/// Constructor
|
||||||
const CameraView(
|
const PokedexView(
|
||||||
{Key? key, required this.resultsCallback, required this.statsCallback})
|
{Key? key, required this.resultsCallback, required this.statsCallback})
|
||||||
: super(key: key);
|
: super(key: key);
|
||||||
|
|
||||||
@override
|
@override
|
||||||
State<CameraView> createState() => _CameraViewState();
|
State<PokedexView> createState() => _PokedexViewState();
|
||||||
}
|
}
|
||||||
|
|
||||||
class _CameraViewState extends State<CameraView> with WidgetsBindingObserver {
|
class _PokedexViewState extends State<PokedexView> with WidgetsBindingObserver {
|
||||||
/// List of available cameras
|
|
||||||
late List<CameraDescription> cameras;
|
late List<CameraDescription> cameras;
|
||||||
|
|
||||||
/// Controller
|
|
||||||
late CameraController cameraController;
|
late CameraController cameraController;
|
||||||
Interpreter? interp;
|
late MLIsolate _mlIsolate;
|
||||||
|
|
||||||
/// true when inference is ongoing
|
/// true when inference is ongoing
|
||||||
bool predicting = false;
|
bool predicting = false;
|
||||||
|
bool _cameraInitialized = false;
|
||||||
|
bool _classifierInitialized = false;
|
||||||
|
|
||||||
late Classifier classy;
|
late Interpreter interpreter;
|
||||||
|
late Classifier classifier;
|
||||||
// /// Instance of [Classifier]
|
|
||||||
// Classifier classifier;
|
|
||||||
//
|
|
||||||
// /// Instance of [IsolateUtils]
|
|
||||||
// IsolateUtils isolateUtils;
|
|
||||||
|
|
||||||
@override
|
@override
|
||||||
void initState() {
|
void initState() {
|
||||||
@@ -54,40 +48,21 @@ class _CameraViewState extends State<CameraView> with WidgetsBindingObserver {
|
|||||||
|
|
||||||
void initStateAsync() async {
|
void initStateAsync() async {
|
||||||
WidgetsBinding.instance.addObserver(this);
|
WidgetsBinding.instance.addObserver(this);
|
||||||
|
_mlIsolate = MLIsolate();
|
||||||
// Spawn a new isolate
|
await _mlIsolate.start();
|
||||||
// isolateUtils = IsolateUtils();
|
|
||||||
// await isolateUtils.start();
|
|
||||||
|
|
||||||
// Camera initialization
|
|
||||||
initializeCamera();
|
initializeCamera();
|
||||||
|
initializeModel();
|
||||||
// final gpuDelegateV2 = GpuDelegateV2(
|
|
||||||
// options: GpuDelegateOptionsV2(
|
|
||||||
// isPrecisionLossAllowed: false,
|
|
||||||
// inferencePreference: TfLiteGpuInferenceUsage.fastSingleAnswer,
|
|
||||||
// inferencePriority1: TfLiteGpuInferencePriority.minLatency,
|
|
||||||
// inferencePriority2: TfLiteGpuInferencePriority.auto,
|
|
||||||
// inferencePriority3: TfLiteGpuInferencePriority.auto,
|
|
||||||
// ));
|
|
||||||
|
|
||||||
|
|
||||||
logger.e("CREATING THE INTERPRETOR");
|
|
||||||
var interpreterOptions = InterpreterOptions();//..addDelegate(gpuDelegateV2);
|
|
||||||
interp = await Interpreter.fromAsset('efficientnet_v2s.tflite',
|
|
||||||
options: interpreterOptions);
|
|
||||||
logger.e("CREATING THE INTERPRETOR");
|
|
||||||
|
|
||||||
classy = Classifier(interpreter: interp);
|
|
||||||
logger.i(interp?.getOutputTensors());
|
|
||||||
// Create an instance of classifier to load model and labels
|
|
||||||
// classifier = Classifier();
|
|
||||||
|
|
||||||
|
|
||||||
// Initially predicting = false
|
|
||||||
predicting = false;
|
predicting = false;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
void initializeModel() async {
|
||||||
|
var interpreterOptions = InterpreterOptions()..threads = 8;
|
||||||
|
interpreter = await Interpreter.fromAsset('efficientnet_v2s.tflite',
|
||||||
|
options: interpreterOptions);
|
||||||
|
classifier = Classifier(interpreter: interpreter);
|
||||||
|
_classifierInitialized = true;
|
||||||
|
}
|
||||||
|
|
||||||
/// Initializes the camera by setting [cameraController]
|
/// Initializes the camera by setting [cameraController]
|
||||||
void initializeCamera() async {
|
void initializeCamera() async {
|
||||||
cameras = await availableCameras();
|
cameras = await availableCameras();
|
||||||
@@ -97,100 +72,61 @@ class _CameraViewState extends State<CameraView> with WidgetsBindingObserver {
|
|||||||
CameraController(cameras[0], ResolutionPreset.low, enableAudio: false);
|
CameraController(cameras[0], ResolutionPreset.low, enableAudio: false);
|
||||||
|
|
||||||
cameraController.initialize().then((_) async {
|
cameraController.initialize().then((_) async {
|
||||||
|
/// previewSize is size of each image frame captured by controller
|
||||||
|
/// 352x288 on iOS, 240p (320x240) on Android with ResolutionPreset.low
|
||||||
// Stream of image passed to [onLatestImageAvailable] callback
|
// Stream of image passed to [onLatestImageAvailable] callback
|
||||||
await cameraController.startImageStream(onLatestImageAvailable);
|
await cameraController.startImageStream(onLatestImageAvailable);
|
||||||
|
setState(() {
|
||||||
/// previewSize is size of each image frame captured by controller
|
_cameraInitialized = true;
|
||||||
///
|
});
|
||||||
/// 352x288 on iOS, 240p (320x240) on Android with ResolutionPreset.low
|
|
||||||
// Size previewSize = cameraController.value.previewSize;
|
|
||||||
//
|
|
||||||
// /// previewSize is size of raw input image to the model
|
|
||||||
// CameraViewSingleton.inputImageSize = previewSize;
|
|
||||||
//
|
|
||||||
// // the display width of image on screen is
|
|
||||||
// // same as screenWidth while maintaining the aspectRatio
|
|
||||||
// Size screenSize = MediaQuery.of(context).size;
|
|
||||||
// CameraViewSingleton.screenSize = screenSize;
|
|
||||||
// CameraViewSingleton.ratio = screenSize.width / previewSize.height;
|
|
||||||
});
|
});
|
||||||
}
|
|
||||||
|
|
||||||
@override
|
|
||||||
Widget build(BuildContext context) {
|
|
||||||
// Return empty container while the camera is not initialized
|
|
||||||
if (!cameraController.value.isInitialized) {
|
|
||||||
return Container();
|
|
||||||
}
|
|
||||||
|
|
||||||
return AspectRatio(
|
|
||||||
aspectRatio: 1/cameraController.value.aspectRatio,
|
|
||||||
child: CameraPreview(cameraController));
|
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Callback to receive each frame [CameraImage] perform inference on it
|
/// Callback to receive each frame [CameraImage] perform inference on it
|
||||||
onLatestImageAvailable(CameraImage cameraImage) async {
|
onLatestImageAvailable(CameraImage cameraImage) async {
|
||||||
// if (classifier.interpreter != null && classifier.labels != null) {
|
if (_classifierInitialized) {
|
||||||
// // If previous inference has not completed then return
|
|
||||||
if (predicting) {
|
if (predicting) {
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
setState(() {
|
setState(() {
|
||||||
predicting = true;
|
predicting = true;
|
||||||
});
|
});
|
||||||
logger.i("RECIEVED IMAGE");
|
var results = await inference(MLIsolateData(
|
||||||
logger.i(cameraImage.format.group);
|
cameraImage, classifier.interpreter.address, classifier.labels));
|
||||||
logger.i(cameraImage);
|
|
||||||
var converted = ImageUtils.convertCameraImage(cameraImage);
|
|
||||||
if (converted != null){
|
|
||||||
|
|
||||||
var result = classy.predict(converted);
|
if (results.containsKey("recognitions")) {
|
||||||
|
widget.resultsCallback(results["recognitions"]);
|
||||||
logger.e("PREDICTED IMAGE");
|
|
||||||
logger.i(result);
|
|
||||||
}
|
}
|
||||||
// logger.i(cameraImage);
|
if (results.containsKey("stats")) {
|
||||||
// logger.i(cameraImage.height);
|
widget.statsCallback(results["stats"]);
|
||||||
// logger.i(cameraImage.width);
|
}
|
||||||
// logger.i(cameraImage.planes[0]);
|
logger.i(results);
|
||||||
//
|
|
||||||
// var uiThreadTimeStart = DateTime.now().millisecondsSinceEpoch;
|
|
||||||
//
|
|
||||||
// // Data to be passed to inference isolate
|
|
||||||
// var isolateData = IsolateData(
|
|
||||||
// cameraImage, classifier.interpreter.address, classifier.labels);
|
|
||||||
//
|
|
||||||
// // We could have simply used the compute method as well however
|
|
||||||
// // it would be as in-efficient as we need to continuously passing data
|
|
||||||
// // to another isolate.
|
|
||||||
//
|
|
||||||
// /// perform inference in separate isolate
|
|
||||||
// Map<String, dynamic> inferenceResults = await inference(isolateData);
|
|
||||||
//
|
|
||||||
// var uiThreadInferenceElapsedTime =
|
|
||||||
// DateTime.now().millisecondsSinceEpoch - uiThreadTimeStart;
|
|
||||||
//
|
|
||||||
// // pass results to HomeView
|
|
||||||
// widget.resultsCallback(inferenceResults["recognitions"]);
|
|
||||||
//
|
|
||||||
// // pass stats to HomeView
|
|
||||||
// widget.statsCallback((inferenceResults["stats"] as Stats)
|
|
||||||
// ..totalElapsedTime = uiThreadInferenceElapsedTime);
|
|
||||||
|
|
||||||
// set predicting to false to allow new frames
|
|
||||||
setState(() {
|
setState(() {
|
||||||
predicting = false;
|
predicting = false;
|
||||||
});
|
});
|
||||||
}
|
}
|
||||||
|
}
|
||||||
|
|
||||||
// /// Runs inference in another isolate
|
@override
|
||||||
// Future<Map<String, dynamic>> inference(IsolateData isolateData) async {
|
Widget build(BuildContext context) {
|
||||||
// ReceivePort responsePort = ReceivePort();
|
// Return empty container while the camera is not initialized
|
||||||
// isolateUtils.sendPort
|
if (!_cameraInitialized) {
|
||||||
// .send(isolateData..responsePort = responsePort.sendPort);
|
return Container();
|
||||||
// var results = await responsePort.first;
|
}
|
||||||
// return results;
|
return AspectRatio(
|
||||||
// }
|
aspectRatio: 1 / cameraController.value.aspectRatio,
|
||||||
|
child: CameraPreview(cameraController));
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Runs inference in another isolate
|
||||||
|
Future<Map<String, dynamic>> inference(MLIsolateData mlIsolateData) async {
|
||||||
|
ReceivePort responsePort = ReceivePort();
|
||||||
|
_mlIsolate.sendPort
|
||||||
|
.send(mlIsolateData..responsePort = responsePort.sendPort);
|
||||||
|
var results = await responsePort.first;
|
||||||
|
return results;
|
||||||
|
}
|
||||||
|
|
||||||
@override
|
@override
|
||||||
void didChangeAppLifecycleState(AppLifecycleState state) async {
|
void didChangeAppLifecycleState(AppLifecycleState state) async {
|
||||||
|
|||||||
@@ -1,26 +1,21 @@
|
|||||||
import 'package:flutter/material.dart';
|
import 'package:flutter/material.dart';
|
||||||
import 'package:tensordex_mobile/ui/poke_view.dart';
|
import 'package:tensordex_mobile/ui/poke_view.dart';
|
||||||
import 'package:tensordex_mobile/utils/recognition.dart';
|
import 'package:tensordex_mobile/tflite/data/recognition.dart';
|
||||||
|
import 'package:tensordex_mobile/tflite/data/stats.dart';
|
||||||
|
|
||||||
import '../utils/logger.dart';
|
|
||||||
|
|
||||||
/// [CameraView] sends each frame for inference
|
/// [PokedexView] sends each frame for inference
|
||||||
class ResultsView extends StatefulWidget {
|
class ResultsView extends StatefulWidget {
|
||||||
|
final List<Recognition> recognitions;
|
||||||
|
final Stats stats;
|
||||||
/// Constructor
|
/// Constructor
|
||||||
const ResultsView({Key? key}) : super(key: key);
|
const ResultsView(this.recognitions, this.stats, {Key? key}) : super(key: key);
|
||||||
|
|
||||||
|
|
||||||
void setResults(Recognition results){
|
|
||||||
logger.i("RESULTS IN THE RESULT VIEW");
|
|
||||||
}
|
|
||||||
|
|
||||||
@override
|
@override
|
||||||
State<ResultsView> createState() => _ResultsViewState();
|
State<ResultsView> createState() => _ResultsViewState();
|
||||||
}
|
}
|
||||||
|
|
||||||
class _ResultsViewState extends State<ResultsView> {
|
class _ResultsViewState extends State<ResultsView> {
|
||||||
|
|
||||||
@override
|
@override
|
||||||
void initState() {
|
void initState() {
|
||||||
super.initState();
|
super.initState();
|
||||||
@@ -28,6 +23,6 @@ class _ResultsViewState extends State<ResultsView> {
|
|||||||
|
|
||||||
@override
|
@override
|
||||||
Widget build(BuildContext context) {
|
Widget build(BuildContext context) {
|
||||||
return Text("data");
|
return Text(widget.recognitions.toString());
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
+14
-100
@@ -3,16 +3,12 @@ import 'package:tensordex_mobile/ui/poke_view.dart';
|
|||||||
import 'package:tensordex_mobile/ui/results_view.dart';
|
import 'package:tensordex_mobile/ui/results_view.dart';
|
||||||
|
|
||||||
import '../utils/logger.dart';
|
import '../utils/logger.dart';
|
||||||
import '../utils/recognition.dart';
|
import '../tflite/data/recognition.dart';
|
||||||
import '../utils/stats.dart';
|
import '../tflite/data/stats.dart';
|
||||||
|
|
||||||
class TensordexHome extends StatefulWidget {
|
class TensordexHome extends StatefulWidget {
|
||||||
const TensordexHome({Key? key, required this.title}) : super(key: key);
|
const TensordexHome({Key? key, required this.title}) : super(key: key);
|
||||||
|
|
||||||
// This widget is the home page of your application. It is stateful, meaning
|
|
||||||
// that it has a State object (defined below) that contains fields that affect
|
|
||||||
// how it looks.
|
|
||||||
|
|
||||||
// This class is the configuration for the state. It holds the values (in this
|
// This class is the configuration for the state. It holds the values (in this
|
||||||
// case the title) provided by the parent (in this case the App widget) and
|
// case the title) provided by the parent (in this case the App widget) and
|
||||||
// used by the build method of the State. Fields in a Widget subclass are
|
// used by the build method of the State. Fields in a Widget subclass are
|
||||||
@@ -25,12 +21,9 @@ class TensordexHome extends StatefulWidget {
|
|||||||
}
|
}
|
||||||
|
|
||||||
class _TensordexHomeState extends State<TensordexHome> {
|
class _TensordexHomeState extends State<TensordexHome> {
|
||||||
|
/// Results from the image classifier
|
||||||
/// Results to draw bounding boxes
|
List<Recognition> results = [Recognition(1, "NOTHING DETECTED", .5)];
|
||||||
List<Recognition>? results;
|
Stats stats = Stats();
|
||||||
|
|
||||||
/// Realtime stats
|
|
||||||
Stats? stats;
|
|
||||||
|
|
||||||
/// Scaffold Key
|
/// Scaffold Key
|
||||||
GlobalKey<ScaffoldState> scaffoldKey = GlobalKey();
|
GlobalKey<ScaffoldState> scaffoldKey = GlobalKey();
|
||||||
@@ -38,106 +31,27 @@ class _TensordexHomeState extends State<TensordexHome> {
|
|||||||
void _incrementCounter() {
|
void _incrementCounter() {
|
||||||
setState(() {
|
setState(() {
|
||||||
logger.d("Counter Incremented!");
|
logger.d("Counter Incremented!");
|
||||||
logger.w("Counter Incremented!");
|
|
||||||
logger.e("Counter Incremented!");
|
|
||||||
});
|
});
|
||||||
}
|
}
|
||||||
|
|
||||||
// void onNewCameraSelected(CameraDescription cameraDescription) async {
|
@override
|
||||||
// final previousCameraController = controller;
|
void initState() {
|
||||||
// // Instantiating the camera controller
|
super.initState();
|
||||||
// final CameraController cameraController = CameraController(
|
}
|
||||||
// cameraDescription,
|
|
||||||
// ResolutionPreset.high,
|
|
||||||
// imageFormatGroup: ImageFormatGroup.jpeg,
|
|
||||||
// );
|
|
||||||
//
|
|
||||||
// // Dispose the previous controller
|
|
||||||
// await previousCameraController.dispose();
|
|
||||||
//
|
|
||||||
// // Replace with the new controller
|
|
||||||
// if (mounted) {
|
|
||||||
// setState(() {
|
|
||||||
// controller = cameraController;
|
|
||||||
// });
|
|
||||||
// }
|
|
||||||
//
|
|
||||||
// // Update UI if controller updated
|
|
||||||
// cameraController.addListener(() {
|
|
||||||
// if (mounted) setState(() {});
|
|
||||||
// });
|
|
||||||
//
|
|
||||||
// // Initialize controller
|
|
||||||
// try {
|
|
||||||
// await cameraController.initialize();
|
|
||||||
// } on CameraException catch (e) {
|
|
||||||
// logger.e('Error initializing camera:', e);
|
|
||||||
// }
|
|
||||||
//
|
|
||||||
// // Update the Boolean
|
|
||||||
// if (mounted) {
|
|
||||||
// setState(() {
|
|
||||||
// _isCameraInitialized = controller.value.isInitialized;
|
|
||||||
// });
|
|
||||||
// }
|
|
||||||
// }
|
|
||||||
|
|
||||||
// @override
|
|
||||||
// void initState() {
|
|
||||||
// super.initState();
|
|
||||||
// WidgetsBinding.instance.addObserver(this);
|
|
||||||
|
|
||||||
// controller = CameraController(_cameras[0], ResolutionPreset.max);
|
|
||||||
// controller.initialize().then((_) {
|
|
||||||
// if (!mounted) {
|
|
||||||
// return;
|
|
||||||
// }
|
|
||||||
//
|
|
||||||
// setState(() {onNewCameraSelected(_cameras[0]);});
|
|
||||||
// }).catchError((Object e) {
|
|
||||||
// if (e is CameraException) {
|
|
||||||
// switch (e.code) {
|
|
||||||
// case 'CameraAccessDenied':
|
|
||||||
// logger.w('User denied camera access.');
|
|
||||||
// controller.initialize().then((_) {
|
|
||||||
// if (!mounted) {
|
|
||||||
// return;
|
|
||||||
// }
|
|
||||||
// setState(() {});
|
|
||||||
// }).catchError((Object e) {
|
|
||||||
// if (e is CameraException) {
|
|
||||||
// switch (e.code) {
|
|
||||||
// case 'CameraAccessDenied':
|
|
||||||
// logger.i('User denied camera access.');
|
|
||||||
// break;
|
|
||||||
// default:
|
|
||||||
// logger.i('Handle other errors.');
|
|
||||||
// break;
|
|
||||||
// }
|
|
||||||
// }
|
|
||||||
// });
|
|
||||||
// break;
|
|
||||||
// default:
|
|
||||||
// logger.i('Handle other errors.');
|
|
||||||
// break;
|
|
||||||
// }
|
|
||||||
// }
|
|
||||||
// });
|
|
||||||
// }
|
|
||||||
|
|
||||||
@override
|
@override
|
||||||
void dispose() {
|
void dispose() {
|
||||||
super.dispose();
|
super.dispose();
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Callback to get inference results from [CameraView]
|
/// Callback to get inference results from [PokedexView]
|
||||||
void resultsCallback(List<Recognition> results) {
|
void resultsCallback(List<Recognition> results) {
|
||||||
setState(() {
|
setState(() {
|
||||||
this.results = results;
|
this.results = results;
|
||||||
});
|
});
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Callback to get inference stats from [CameraView]
|
/// Callback to get inference stats from [PokedexView]
|
||||||
void statsCallback(Stats stats) {
|
void statsCallback(Stats stats) {
|
||||||
setState(() {
|
setState(() {
|
||||||
this.stats = stats;
|
this.stats = stats;
|
||||||
@@ -152,12 +66,12 @@ class _TensordexHomeState extends State<TensordexHome> {
|
|||||||
),
|
),
|
||||||
body: Center(
|
body: Center(
|
||||||
child: Column(
|
child: Column(
|
||||||
mainAxisAlignment: MainAxisAlignment.center,
|
mainAxisAlignment: MainAxisAlignment.start,
|
||||||
children: <Widget>[
|
children: <Widget>[
|
||||||
CameraView(
|
PokedexView(
|
||||||
resultsCallback: resultsCallback,
|
resultsCallback: resultsCallback,
|
||||||
statsCallback: statsCallback),
|
statsCallback: statsCallback),
|
||||||
const ResultsView(),
|
ResultsView(results, stats),
|
||||||
],
|
],
|
||||||
),
|
),
|
||||||
),
|
),
|
||||||
|
|||||||
@@ -1,10 +0,0 @@
|
|||||||
import 'dart:ui';
|
|
||||||
|
|
||||||
class CameraViewSingleton {
|
|
||||||
static double ratio = 0.0;
|
|
||||||
static Size screenSize = const Size(0, 0);
|
|
||||||
static Size inputImageSize = const Size(0, 0);
|
|
||||||
|
|
||||||
static Size get actualPreviewSize =>
|
|
||||||
Size(screenSize.width, screenSize.width * ratio);
|
|
||||||
}
|
|
||||||
@@ -1,23 +0,0 @@
|
|||||||
/// Bundles different elapsed times
|
|
||||||
class Stats {
|
|
||||||
/// Total time taken in the isolate where the inference runs
|
|
||||||
int totalPredictTime;
|
|
||||||
|
|
||||||
/// [totalPredictTime] + communication overhead time
|
|
||||||
/// between main isolate and another isolate
|
|
||||||
int totalElapsedTime;
|
|
||||||
|
|
||||||
/// Time for which inference runs
|
|
||||||
int inferenceTime;
|
|
||||||
|
|
||||||
/// Time taken to pre-process the image
|
|
||||||
int preProcessingTime;
|
|
||||||
|
|
||||||
Stats(this.totalPredictTime, this.totalElapsedTime, this.inferenceTime,
|
|
||||||
this.preProcessingTime);
|
|
||||||
|
|
||||||
@override
|
|
||||||
String toString() {
|
|
||||||
return 'Stats{totalPredictTime: $totalPredictTime, totalElapsedTime: $totalElapsedTime, inferenceTime: $inferenceTime, preProcessingTime: $preProcessingTime}';
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@@ -0,0 +1,81 @@
|
|||||||
|
|
||||||
|
// void onNewCameraSelected(CameraDescription cameraDescription) async {
|
||||||
|
// final previousCameraController = controller;
|
||||||
|
// // Instantiating the camera controller
|
||||||
|
// final CameraController cameraController = CameraController(
|
||||||
|
// cameraDescription,
|
||||||
|
// ResolutionPreset.high,
|
||||||
|
// imageFormatGroup: ImageFormatGroup.jpeg,
|
||||||
|
// );
|
||||||
|
//
|
||||||
|
// // Dispose the previous controller
|
||||||
|
// await previousCameraController.dispose();
|
||||||
|
//
|
||||||
|
// // Replace with the new controller
|
||||||
|
// if (mounted) {
|
||||||
|
// setState(() {
|
||||||
|
// controller = cameraController;
|
||||||
|
// });
|
||||||
|
// }
|
||||||
|
//
|
||||||
|
// // Update UI if controller updated
|
||||||
|
// cameraController.addListener(() {
|
||||||
|
// if (mounted) setState(() {});
|
||||||
|
// });
|
||||||
|
//
|
||||||
|
// // Initialize controller
|
||||||
|
// try {
|
||||||
|
// await cameraController.initialize();
|
||||||
|
// } on CameraException catch (e) {
|
||||||
|
// logger.e('Error initializing camera:', e);
|
||||||
|
// }
|
||||||
|
//
|
||||||
|
// // Update the Boolean
|
||||||
|
// if (mounted) {
|
||||||
|
// setState(() {
|
||||||
|
// _isCameraInitialized = controller.value.isInitialized;
|
||||||
|
// });
|
||||||
|
// }
|
||||||
|
// }
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
// WidgetsBinding.instance.addObserver(this);
|
||||||
|
|
||||||
|
// controller = CameraController(_cameras[0], ResolutionPreset.max);
|
||||||
|
// controller.initialize().then((_) {
|
||||||
|
// if (!mounted) {
|
||||||
|
// return;
|
||||||
|
// }
|
||||||
|
//
|
||||||
|
// setState(() {onNewCameraSelected(_cameras[0]);});
|
||||||
|
// }).catchError((Object e) {
|
||||||
|
// if (e is CameraException) {
|
||||||
|
// switch (e.code) {
|
||||||
|
// case 'CameraAccessDenied':
|
||||||
|
// logger.w('User denied camera access.');
|
||||||
|
// controller.initialize().then((_) {
|
||||||
|
// if (!mounted) {
|
||||||
|
// return;
|
||||||
|
// }
|
||||||
|
// setState(() {});
|
||||||
|
// }).catchError((Object e) {
|
||||||
|
// if (e is CameraException) {
|
||||||
|
// switch (e.code) {
|
||||||
|
// case 'CameraAccessDenied':
|
||||||
|
// logger.i('User denied camera access.');
|
||||||
|
// break;
|
||||||
|
// default:
|
||||||
|
// logger.i('Handle other errors.');
|
||||||
|
// break;
|
||||||
|
// }
|
||||||
|
// }
|
||||||
|
// });
|
||||||
|
// break;
|
||||||
|
// default:
|
||||||
|
// logger.i('Handle other errors.');
|
||||||
|
// break;
|
||||||
|
// }
|
||||||
|
// }
|
||||||
|
// });
|
||||||
|
// }
|
||||||
Reference in New Issue
Block a user