diff --git a/assets/efficientnet_v2s.tflite b/assets/efficientnet_v2s.tflite new file mode 100644 index 0000000..9fe20d0 Binary files /dev/null and b/assets/efficientnet_v2s.tflite differ diff --git a/assets/labels.txt b/assets/labels.txt new file mode 100644 index 0000000..cf674ce --- /dev/null +++ b/assets/labels.txt @@ -0,0 +1,807 @@ +abomasnow +abra +absol +accelgor +aegislash +aerodactyl +aggron +aipom +alakazam +alomomola +altaria +amaura +ambipom +amoonguss +ampharos +anorith +araquanid +arbok +arcanine +arceus +archen +archeops +ariados +armaldo +aromatisse +aron +articuno +audino +aurorus +avalugg +axew +azelf +azumarill +azurill +bagon +baltoy +banette +barbaracle +barboach +basculin +bastiodon +bayleef +beartic +beautifly +beedrill +beheeyem +beldum +bellossom +bellsprout +bergmite +bewear +bibarel +bidoof +binacle +bisharp +blacephalon +blastoise +blaziken +blissey +blitzle +boldore +bonsly +bouffalant +bounsweet +braixen +braviary +breloom +brionne +bronzong +bronzor +bruxish +budew +buizel +bulbasaur +buneary +bunnelby +burmy +butterfree +buzzwole +cacnea +cacturne +camerupt +carbink +carnivine +carracosta +carvanha +cascoon +castform +caterpie +celebi +celesteela +chandelure +chansey +charizard +charjabug +charmander +charmeleon +chatot +cherrim +cherubi +chesnaught +chespin +chikorita +chimchar +chimecho +chinchou +chingling +cinccino +clamperl +clauncher +clawitzer +claydol +clefable +clefairy +cleffa +cloyster +cobalion +cofagrigus +combee +combusken +comfey +conkeldurr +corphish +corsola +cosmoem +cosmog +cottonee +crabominable +crabrawler +cradily +cranidos +crawdaunt +cresselia +croagunk +crobat +croconaw +crustle +cryogonal +cubchoo +cubone +cutiefly +cyndaquil +darkrai +darmanitan +dartrix +darumaka +decidueye +dedenne +deerling +deino +delcatty +delibird +delphox +deoxys +dewgong +dewott +dewpider +dhelmise +dialga +diancie +diggersby +diglett +ditto +dodrio +doduo +donphan +doublade +dragalge +dragonair +dragonite +drampa +drapion +dratini +drifblim +drifloon +drilbur +drowzee +druddigon +ducklett +dugtrio +dunsparce +duosion +durant +dusclops +dusknoir +duskull +dustox +dwebble +eelektrik +eelektross +eevee +ekans +electabuzz +electivire +electrike +electrode +elekid +elgyem +emboar +emolga +empoleon +entei +escavalier +espeon +espurr +excadrill +exeggcute +exeggutor +exploud +farfetchd +fearow +feebas +fennekin +feraligatr +ferroseed +ferrothorn +finneon +flaaffy +flabebe +flareon +fletchinder +fletchling +floatzel +floette +florges +flygon +fomantis +foongus +forretress +fraxure +frillish +froakie +frogadier +froslass +furfrou +furret +gabite +gallade +galvantula +garbodor +garchomp +gardevoir +gastly +gastrodon +genesect +gengar +geodude +gible +gigalith +girafarig +giratina +glaceon +glalie +glameow +gligar +gliscor +gloom +gogoat +golbat +goldeen +golduck +golem +golett +golisopod +golurk +goodra +goomy +gorebyss +gothita +gothitelle +gothorita +gourgeist +granbull +graveler +greninja +grimer +grotle +groudon +grovyle +growlithe +grubbin +grumpig +gulpin +gumshoos +gurdurr +guzzlord +gyarados +hakamo-o +happiny +hariyama +haunter +hawlucha +haxorus +heatmor +heatran +heliolisk +helioptile +heracross +herdier +hippopotas +hippowdon +hitmonchan +hitmonlee +hitmontop +ho-oh +honchkrow +honedge +hoopa +hoothoot +hoppip +horsea +houndoom +houndour +huntail +hydreigon +hypno +igglybuff +illumise +incineroar +infernape +inkay +ivysaur +jangmo-o +jellicent +jigglypuff +jirachi +jolteon +joltik +jumpluff +jynx +kabuto +kabutops +kadabra +kakuna +kangaskhan +karrablast +kartana +kecleon +keldeo +kingdra +kingler +kirlia +klang +klefki +klink +klinklang +koffing +komala +kommo-o +krabby +kricketot +kricketune +krokorok +krookodile +kyogre +kyurem +lairon +lampent +landorus +lanturn +lapras +larvesta +larvitar +latias +latios +leafeon +leavanny +ledian +ledyba +lickilicky +lickitung +liepard +lileep +lilligant +lillipup +linoone +litleo +litten +litwick +lombre +lopunny +lotad +loudred +lucario +ludicolo +lugia +lumineon +lunala +lunatone +lurantis +luvdisc +luxio +luxray +lycanroc +machamp +machoke +machop +magby +magcargo +magearna +magikarp +magmar +magmortar +magnemite +magneton +magnezone +makuhita +malamar +mamoswine +manaphy +mandibuzz +manectric +mankey +mantine +mantyke +maractus +mareanie +mareep +marill +marowak +marshadow +marshtomp +masquerain +mawile +medicham +meditite +meganium +meloetta +meowstic +meowth +mesprit +metagross +metang +metapod +mew +mewtwo +mienfoo +mienshao +mightyena +milotic +miltank +mime-jr +mimikyu +minccino +minior +minun +misdreavus +mismagius +moltres +monferno +morelull +mothim +mr-mime +mudbray +mudkip +mudsdale +muk +munchlax +munna +murkrow +musharna +naganadel +natu +necrozma +nidoking +nidoqueen +nidoran-f +nidoran-m +nidorina +nidorino +nihilego +nincada +ninetales +ninjask +noctowl +noibat +noivern +nosepass +numel +nuzleaf +octillery +oddish +omanyte +omastar +onix +oranguru +oricorio +oshawott +pachirisu +palkia +palossand +palpitoad +pancham +pangoro +panpour +pansage +pansear +paras +parasect +passimian +patrat +pawniard +pelipper +persian +petilil +phanpy +phantump +pheromosa +phione +pichu +pidgeot +pidgeotto +pidgey +pidove +pignite +pikachu +pikipek +piloswine +pineco +pinsir +piplup +plusle +poipole +politoed +poliwag +poliwhirl +poliwrath +ponyta +poochyena +popplio +porygon +porygon-z +porygon2 +primarina +primeape +prinplup +probopass +psyduck +pumpkaboo +pupitar +purrloin +purugly +pyroar +pyukumuku +quagsire +quilava +quilladin +qwilfish +raichu +raikou +ralts +rampardos +rapidash +raticate +rattata +rayquaza +regice +regigigas +regirock +registeel +relicanth +remoraid +reshiram +reuniclus +rhydon +rhyhorn +rhyperior +ribombee +riolu +rockruff +roggenrola +roselia +roserade +rotom +rowlet +rufflet +sableye +salamence +salandit +salazzle +samurott +sandile +sandshrew +sandslash +sandygast +sawk +sawsbuck +scatterbug +sceptile +scizor +scolipede +scrafty +scraggy +scyther +seadra +seaking +sealeo +seedot +seel +seismitoad +sentret +serperior +servine +seviper +sewaddle +sharpedo +shaymin +shedinja +shelgon +shellder +shellos +shelmet +shieldon +shiftry +shiinotic +shinx +shroomish +shuckle +shuppet +sigilyph +silcoon +silvally +simipour +simisage +simisear +skarmory +skiddo +skiploom +skitty +skorupi +skrelp +skuntank +slaking +slakoth +sliggoo +slowbro +slowking +slowpoke +slugma +slurpuff +smeargle +smoochum +sneasel +snivy +snorlax +snorunt +snover +snubbull +solgaleo +solosis +solrock +spearow +spewpa +spheal +spinarak +spinda +spiritomb +spoink +spritzee +squirtle +stakataka +stantler +staraptor +staravia +starly +starmie +staryu +steelix +steenee +stoutland +stufful +stunfisk +stunky +sudowoodo +suicune +sunflora +sunkern +surskit +swablu +swadloon +swalot +swampert +swanna +swellow +swinub +swirlix +swoobat +sylveon +taillow +talonflame +tangela +tangrowth +tapu-bulu +tapu-fini +tapu-koko +tapu-lele +tauros +teddiursa +tentacool +tentacruel +tepig +terrakion +throh +thundurus +timburr +tirtouga +togedemaru +togekiss +togepi +togetic +torchic +torkoal +tornadus +torracat +torterra +totodile +toucannon +toxapex +toxicroak +tranquill +trapinch +treecko +trevenant +tropius +trubbish +trumbeak +tsareena +turtonator +turtwig +tympole +tynamo +type-null +typhlosion +tyranitar +tyrantrum +tyrogue +tyrunt +umbreon +unfezant +unown +ursaring +uxie +vanillish +vanillite +vanilluxe +vaporeon +venipede +venomoth +venonat +venusaur +vespiquen +vibrava +victini +victreebel +vigoroth +vikavolt +vileplume +virizion +vivillon +volbeat +volcanion +volcarona +voltorb +vullaby +vulpix +wailmer +wailord +walrein +wartortle +watchog +weavile +weedle +weepinbell +weezing +whimsicott +whirlipede +whiscash +whismur +wigglytuff +wimpod +wingull +wishiwashi +wobbuffet +woobat +wooper +wormadam +wurmple +wynaut +xatu +xerneas +xurkitree +yamask +yanma +yanmega +yungoos +yveltal +zangoose +zapdos +zebstrika +zekrom +zeraora +zigzagoon +zoroark +zorua +zubat +zweilous +zygarde diff --git a/lib/main.dart b/lib/main.dart index d62427d..dc3ea3d 100644 --- a/lib/main.dart +++ b/lib/main.dart @@ -1,22 +1,8 @@ import 'package:flutter/material.dart'; -import 'package:logger/logger.dart'; -import 'package:camera/camera.dart'; - -var logger = Logger( - printer: PrettyPrinter(), -); - -late List _cameras; +import 'package:tensordex_mobile/ui/home.dart'; +import 'package:tensordex_mobile/utils/logger.dart'; Future main() async { - try { - WidgetsFlutterBinding.ensureInitialized(); - _cameras = await availableCameras(); - } on CameraException catch (e) { - logger.e('Error in fetching the cameras:', e); - } - - _cameras = await availableCameras(); runApp(const MyApp()); } @@ -26,6 +12,7 @@ class MyApp extends StatelessWidget { // This widget is the root of your application. @override Widget build(BuildContext context) { + logger.i("Building main app"); return MaterialApp( title: 'Tensordex', theme: ThemeData( @@ -44,227 +31,3 @@ class MyApp extends StatelessWidget { ); } } - -class TensordexHome extends StatefulWidget { - const TensordexHome({Key? key, required this.title}) : super(key: key); - - // This widget is the home page of your application. It is stateful, meaning - // that it has a State object (defined below) that contains fields that affect - // how it looks. - - // This class is the configuration for the state. It holds the values (in this - // case the title) provided by the parent (in this case the App widget) and - // used by the build method of the State. Fields in a Widget subclass are - // always marked "final". - - final String title; - - @override - State createState() => _TensordexHomeState(); -} - -class _TensordexHomeState extends State with WidgetsBindingObserver { - int _counter = 0; - late CameraController controller; - bool _isCameraInitialized = false; - - void _incrementCounter() { - setState(() { - // This call to setState tells the Flutter framework that something has - // changed in this State, which causes it to rerun the build method below - // so that the display can reflect the updated values. If we changed - // _counter without calling setState(), then the build method would not be - // called again, and so nothing would appear to happen. - _counter++; - logger.d("Counter Incremented!"); - logger.w("Counter Incremented!"); - logger.e("Counter Incremented!"); - }); - } - - void onNewCameraSelected(CameraDescription cameraDescription) async { - final previousCameraController = controller; - // Instantiating the camera controller - final CameraController cameraController = CameraController( - cameraDescription, - ResolutionPreset.high, - imageFormatGroup: ImageFormatGroup.jpeg, - ); - - // Dispose the previous controller - await previousCameraController.dispose(); - - // Replace with the new controller - if (mounted) { - setState(() { - controller = cameraController; - }); - } - - // Update UI if controller updated - cameraController.addListener(() { - if (mounted) setState(() {}); - }); - - // Initialize controller - try { - await cameraController.initialize(); - } on CameraException catch (e) { - logger.e('Error initializing camera:', e); - } - - // Update the Boolean - if (mounted) { - setState(() { - _isCameraInitialized = controller.value.isInitialized; - }); - } - } - - - @override - void didChangeAppLifecycleState(AppLifecycleState state) { - final CameraController cameraController = controller; - - switch (state) { - case AppLifecycleState.resumed: - // -- - logger.i('Resumed'); - break; - case AppLifecycleState.inactive: - // -- - logger.i('Inactive'); - break; - case AppLifecycleState.paused: - // -- - logger.i('Paused'); - break; - case AppLifecycleState.detached: - // -- - logger.i('Detached'); - break; - } - // App state changed before we got the chance to initialize. - if (!cameraController.value.isInitialized) { - return; - } - - if (state == AppLifecycleState.inactive) { - // Free up memory when camera not active - cameraController.dispose(); - } else if (state == AppLifecycleState.resumed) { - // Reinitialize the camera with same properties - onNewCameraSelected(cameraController.description); - } - } - @override - void initState() { - super.initState(); - WidgetsBinding.instance.addObserver(this); - - - controller = CameraController(_cameras[0], ResolutionPreset.max); - controller.initialize().then((_) { - if (!mounted) { - return; - } - - setState(() {onNewCameraSelected(_cameras[0]);}); - }).catchError((Object e) { - if (e is CameraException) { - switch (e.code) { - case 'CameraAccessDenied': - logger.w('User denied camera access.'); - controller.initialize().then((_) { - if (!mounted) { - return; - } - setState(() {}); - }).catchError((Object e) { - if (e is CameraException) { - switch (e.code) { - case 'CameraAccessDenied': - logger.i('User denied camera access.'); - break; - default: - logger.i('Handle other errors.'); - break; - } - } - }); - break; - default: - logger.i('Handle other errors.'); - break; - } - } - }); - } - - @override - void dispose() { - controller.dispose(); - WidgetsBinding.instance.removeObserver(this); - super.dispose(); - } - - @override - Widget build(BuildContext context) { - // This method is rerun every time setState is called, for instance as done - // by the _incrementCounter method above. - // - // The Flutter framework has been optimized to make rerunning build methods - // fast, so that you can just rebuild anything that needs updating rather - // than having to individually change instances of widgets. - - return Scaffold( - appBar: AppBar( - // Here we take the value from the MyHomePage object that was created by - // the App.build method, and use it to set our appbar title. - title: Text(widget.title), - ), - body: Center( - // Center is a layout widget. It takes a single child and positions it - // in the middle of the parent. - child: Column( - // Column is also a layout widget. It takes a list of children and - // arranges them vertically. By default, it sizes itself to fit its - // children horizontally, and tries to be as tall as its parent. - // - // Invoke "debug painting" (press "p" in the console, choose the - // "Toggle Debug Paint" action from the Flutter Inspector in Android - // Studio, or the "Toggle Debug Paint" command in Visual Studio Code) - // to see the wireframe for each widget. - // - // Column has various properties to control how it sizes itself and - // how it positions its children. Here we use mainAxisAlignment to - // center the children vertically; the main axis here is the vertical - // axis because Columns are vertical (the cross axis would be - // horizontal). - mainAxisAlignment: MainAxisAlignment.center, - children: [ - const Text( - 'You have pushed the button this many times:', - ), - Text( - '$_counter', - style: Theme.of(context).textTheme.headline4, - ), - controller.value.isInitialized - ? CameraPreview(controller) - : const Text("Please enable the camera!"), - ], - ), - ), - floatingActionButton: GestureDetector( - onLongPress: () { - _incrementCounter(); - }, - child: FloatingActionButton( - onPressed: _incrementCounter, - tooltip: 'Increment', - child: const Icon(Icons.photo_camera), - ), // This trailing comma makes auto-formatting nicer for build methods. - )); - } -} diff --git a/lib/poke_view.dart b/lib/poke_view.dart deleted file mode 100644 index e69de29..0000000 diff --git a/lib/ui/home.dart b/lib/ui/home.dart new file mode 100644 index 0000000..e2ee1c8 --- /dev/null +++ b/lib/ui/home.dart @@ -0,0 +1,186 @@ +import 'package:flutter/material.dart'; +import 'package:camera/camera.dart'; +import 'package:tensordex_mobile/ui/poke_view.dart'; + +import '../utils/logger.dart'; +import '../utils/recognition.dart'; +import '../utils/stats.dart'; + +class TensordexHome extends StatefulWidget { + const TensordexHome({Key? key, required this.title}) : super(key: key); + + // This widget is the home page of your application. It is stateful, meaning + // that it has a State object (defined below) that contains fields that affect + // how it looks. + + // This class is the configuration for the state. It holds the values (in this + // case the title) provided by the parent (in this case the App widget) and + // used by the build method of the State. Fields in a Widget subclass are + // always marked "final". + + final String title; + + @override + State createState() => _TensordexHomeState(); +} + +class _TensordexHomeState extends State { + int _counter = 0; + + /// Results to draw bounding boxes + List? results; + + /// Realtime stats + Stats? stats; + + /// Scaffold Key + GlobalKey scaffoldKey = GlobalKey(); + + void _incrementCounter() { + setState(() { + _counter++; + logger.d("Counter Incremented!"); + logger.w("Counter Incremented!"); + logger.e("Counter Incremented!"); + }); + } + + // void onNewCameraSelected(CameraDescription cameraDescription) async { + // final previousCameraController = controller; + // // Instantiating the camera controller + // final CameraController cameraController = CameraController( + // cameraDescription, + // ResolutionPreset.high, + // imageFormatGroup: ImageFormatGroup.jpeg, + // ); + // + // // Dispose the previous controller + // await previousCameraController.dispose(); + // + // // Replace with the new controller + // if (mounted) { + // setState(() { + // controller = cameraController; + // }); + // } + // + // // Update UI if controller updated + // cameraController.addListener(() { + // if (mounted) setState(() {}); + // }); + // + // // Initialize controller + // try { + // await cameraController.initialize(); + // } on CameraException catch (e) { + // logger.e('Error initializing camera:', e); + // } + // + // // Update the Boolean + // if (mounted) { + // setState(() { + // _isCameraInitialized = controller.value.isInitialized; + // }); + // } + // } + + // @override + // void initState() { + // super.initState(); + // WidgetsBinding.instance.addObserver(this); + + // controller = CameraController(_cameras[0], ResolutionPreset.max); + // controller.initialize().then((_) { + // if (!mounted) { + // return; + // } + // + // setState(() {onNewCameraSelected(_cameras[0]);}); + // }).catchError((Object e) { + // if (e is CameraException) { + // switch (e.code) { + // case 'CameraAccessDenied': + // logger.w('User denied camera access.'); + // controller.initialize().then((_) { + // if (!mounted) { + // return; + // } + // setState(() {}); + // }).catchError((Object e) { + // if (e is CameraException) { + // switch (e.code) { + // case 'CameraAccessDenied': + // logger.i('User denied camera access.'); + // break; + // default: + // logger.i('Handle other errors.'); + // break; + // } + // } + // }); + // break; + // default: + // logger.i('Handle other errors.'); + // break; + // } + // } + // }); + // } + + @override + void dispose() { + // controller.dispose(); + // WidgetsBinding.instance.removeObserver(this); + super.dispose(); + } + + /// Callback to get inference results from [CameraView] + void resultsCallback(List results) { + setState(() { + this.results = results; + }); + } + + /// Callback to get inference stats from [CameraView] + void statsCallback(Stats stats) { + setState(() { + this.stats = stats; + }); + } + + @override + Widget build(BuildContext context) { + return Scaffold( + appBar: AppBar( + title: Text(widget.title), + ), + body: Center( + child: Column( + mainAxisAlignment: MainAxisAlignment.center, + children: [ + const Text( + 'You have pushed the button this many times:', + ), + Text( + '$_counter', + style: Theme.of(context).textTheme.headline4, + ), + CameraView( + resultsCallback: resultsCallback, + statsCallback: statsCallback + ), + ], + ), + ), + floatingActionButton: GestureDetector( + onLongPress: () { + _incrementCounter(); + }, + child: FloatingActionButton( + onPressed: _incrementCounter, + tooltip: 'Increment', + child: const Icon(Icons.photo_camera), + ), // This trailing comma makes auto-formatting nicer for build methods. + )); + } +} diff --git a/lib/ui/poke_view.dart b/lib/ui/poke_view.dart new file mode 100644 index 0000000..b32ff75 --- /dev/null +++ b/lib/ui/poke_view.dart @@ -0,0 +1,181 @@ +import 'dart:isolate'; + +import 'package:camera/camera.dart'; +import 'package:flutter/material.dart'; + +import '../utils/logger.dart'; +import '../utils/recognition.dart'; +import '../utils/stats.dart'; + +/// [CameraView] sends each frame for inference +class CameraView extends StatefulWidget { + /// Callback to pass results after inference to [HomeView] + final Function(List recognitions) resultsCallback; + + /// Callback to inference stats to [HomeView] + final Function(Stats stats) statsCallback; + + /// Constructor + const CameraView( + {Key? key, required this.resultsCallback, required this.statsCallback}) + : super(key: key); + + @override + State createState() => _CameraViewState(); +} + +class _CameraViewState extends State with WidgetsBindingObserver { + /// List of available cameras + late List cameras; + + /// Controller + late CameraController cameraController; + + /// true when inference is ongoing + bool predicting = false; + + // /// Instance of [Classifier] + // Classifier classifier; + // + // /// Instance of [IsolateUtils] + // IsolateUtils isolateUtils; + + @override + void initState() { + initStateAsync(); + super.initState(); + } + + void initStateAsync() async { + WidgetsBinding.instance.addObserver(this); + + // Spawn a new isolate + // isolateUtils = IsolateUtils(); + // await isolateUtils.start(); + + // Camera initialization + initializeCamera(); + + // Create an instance of classifier to load model and labels + // classifier = Classifier(); + + // Initially predicting = false + predicting = false; + } + + /// Initializes the camera by setting [cameraController] + void initializeCamera() async { + cameras = await availableCameras(); + + // cameras[0] for rear-camera + cameraController = + CameraController(cameras[0], ResolutionPreset.low, enableAudio: false); + + cameraController.initialize().then((_) async { + // Stream of image passed to [onLatestImageAvailable] callback + await cameraController.startImageStream(onLatestImageAvailable); + + /// previewSize is size of each image frame captured by controller + /// + /// 352x288 on iOS, 240p (320x240) on Android with ResolutionPreset.low + // Size previewSize = cameraController.value.previewSize; + // + // /// previewSize is size of raw input image to the model + // CameraViewSingleton.inputImageSize = previewSize; + // + // // the display width of image on screen is + // // same as screenWidth while maintaining the aspectRatio + // Size screenSize = MediaQuery.of(context).size; + // CameraViewSingleton.screenSize = screenSize; + // CameraViewSingleton.ratio = screenSize.width / previewSize.height; + }); + } + + @override + Widget build(BuildContext context) { + // Return empty container while the camera is not initialized + if (!cameraController.value.isInitialized || cameraController == null) { + return Container(); + } + + return AspectRatio( + aspectRatio: 1/cameraController.value.aspectRatio, + child: CameraPreview(cameraController)); + } + + /// Callback to receive each frame [CameraImage] perform inference on it + onLatestImageAvailable(CameraImage cameraImage) async { + // if (classifier.interpreter != null && classifier.labels != null) { + // // If previous inference has not completed then return + if (predicting) { + return; + } + setState(() { + predicting = true; + }); + logger.i("RECIEVED IMAGE"); + // logger.i(cameraImage); + // logger.i(cameraImage.height); + // logger.i(cameraImage.width); + // logger.i(cameraImage.planes[0]); + // + // var uiThreadTimeStart = DateTime.now().millisecondsSinceEpoch; + // + // // Data to be passed to inference isolate + // var isolateData = IsolateData( + // cameraImage, classifier.interpreter.address, classifier.labels); + // + // // We could have simply used the compute method as well however + // // it would be as in-efficient as we need to continuously passing data + // // to another isolate. + // + // /// perform inference in separate isolate + // Map inferenceResults = await inference(isolateData); + // + // var uiThreadInferenceElapsedTime = + // DateTime.now().millisecondsSinceEpoch - uiThreadTimeStart; + // + // // pass results to HomeView + // widget.resultsCallback(inferenceResults["recognitions"]); + // + // // pass stats to HomeView + // widget.statsCallback((inferenceResults["stats"] as Stats) + // ..totalElapsedTime = uiThreadInferenceElapsedTime); + + // set predicting to false to allow new frames + setState(() { + predicting = false; + }); + } + +// /// Runs inference in another isolate +// Future> inference(IsolateData isolateData) async { +// ReceivePort responsePort = ReceivePort(); +// isolateUtils.sendPort +// .send(isolateData..responsePort = responsePort.sendPort); +// var results = await responsePort.first; +// return results; +// } + + @override + void didChangeAppLifecycleState(AppLifecycleState state) async { + switch (state) { + case AppLifecycleState.paused: + cameraController.stopImageStream(); + break; + case AppLifecycleState.resumed: + if (!cameraController.value.isStreamingImages) { + await cameraController.startImageStream(onLatestImageAvailable); + } + break; + default: + } + } + + @override + void dispose() { + WidgetsBinding.instance.removeObserver(this); + cameraController.dispose(); + super.dispose(); + } +} diff --git a/lib/camera_view.dart b/lib/ui/results_view.dart similarity index 100% rename from lib/camera_view.dart rename to lib/ui/results_view.dart diff --git a/lib/utils/camera_view_singleton.dart b/lib/utils/camera_view_singleton.dart new file mode 100644 index 0000000..562fd87 --- /dev/null +++ b/lib/utils/camera_view_singleton.dart @@ -0,0 +1,10 @@ +import 'dart:ui'; + +class CameraViewSingleton { + static double ratio = 0.0; + static Size screenSize = const Size(0, 0); + static Size inputImageSize = const Size(0, 0); + + static Size get actualPreviewSize => + Size(screenSize.width, screenSize.width * ratio); +} diff --git a/lib/utils/image_utils.dart b/lib/utils/image_utils.dart new file mode 100644 index 0000000..bb0ce04 --- /dev/null +++ b/lib/utils/image_utils.dart @@ -0,0 +1,86 @@ +import 'dart:io'; + +import 'package:camera/camera.dart'; +import 'package:image/image.dart' as image_lib; +import 'package:logger/logger.dart'; +import 'package:path_provider/path_provider.dart'; + +import 'logger.dart'; + +/// ImageUtils +class ImageUtils { + /// Converts a [CameraImage] in YUV420 format to [image_lib.Image] in RGB format + static image_lib.Image? convertCameraImage(CameraImage cameraImage) { + if (cameraImage.format.group == ImageFormatGroup.yuv420) { + return convertYUV420ToImage(cameraImage); + } else if (cameraImage.format.group == ImageFormatGroup.bgra8888) { + return convertBGRA8888ToImage(cameraImage); + } else { + return null; + } + } + + /// Converts a [CameraImage] in BGRA888 format to [image_lib.Image] in RGB format + static image_lib.Image convertBGRA8888ToImage(CameraImage cameraImage) { + image_lib.Image img = image_lib.Image.fromBytes( + cameraImage.planes[0].width ?? 0, + cameraImage.planes[0].height ?? 0, + cameraImage.planes[0].bytes, + format: image_lib.Format.bgra + ); + return img; + } + + /// Converts a [CameraImage] in YUV420 format to [image_lib.Image] in RGB format + static image_lib.Image convertYUV420ToImage(CameraImage cameraImage) { + final int width = cameraImage.width; + final int height = cameraImage.height; + + final int uvRowStride = cameraImage.planes[1].bytesPerRow; + final int uvPixelStride = cameraImage.planes[1].bytesPerPixel?? 0; + + final image = image_lib.Image(width, height); + + for (int w = 0; w < width; w++) { + for (int h = 0; h < height; h++) { + final int uvIndex = + uvPixelStride * (w / 2).floor() + uvRowStride * (h / 2).floor(); + final int index = h * width + w; + + final y = cameraImage.planes[0].bytes[index]; + final u = cameraImage.planes[1].bytes[uvIndex]; + final v = cameraImage.planes[2].bytes[uvIndex]; + + image.data[index] = ImageUtils.yuv2rgb(y, u, v); + } + } + return image; + } + + /// Convert a single YUV pixel to RGB + static int yuv2rgb(int y, int u, int v) { + // Convert yuv pixel to rgb + int r = (y + v * 1436 / 1024 - 179).round(); + int g = (y - u * 46549 / 131072 + 44 - v * 93604 / 131072 + 91).round(); + int b = (y + u * 1814 / 1024 - 227).round(); + + // Clipping RGB values to be inside boundaries [ 0 , 255 ] + r = r.clamp(0, 255); + g = g.clamp(0, 255); + b = b.clamp(0, 255); + + return 0xff000000 | + ((b << 16) & 0xff0000) | + ((g << 8) & 0xff00) | + (r & 0xff); + } + + static void saveImage(image_lib.Image image, [int i = 0]) async { + List jpeg = image_lib.JpegEncoder().encodeImage(image); + final appDir = await getTemporaryDirectory(); + final appPath = appDir.path; + final fileOnDevice = File('$appPath/out$i.jpg'); + await fileOnDevice.writeAsBytes(jpeg, flush: true); + logger.i('Saved $appPath/out$i.jpg'); + } +} diff --git a/lib/utils/logger.dart b/lib/utils/logger.dart new file mode 100644 index 0000000..dc80cb7 --- /dev/null +++ b/lib/utils/logger.dart @@ -0,0 +1,5 @@ +import 'package:logger/logger.dart'; + +var logger = Logger( + printer: PrettyPrinter(), +); diff --git a/lib/utils/recognition.dart b/lib/utils/recognition.dart new file mode 100644 index 0000000..bcd5166 --- /dev/null +++ b/lib/utils/recognition.dart @@ -0,0 +1,21 @@ + +/// Represents the recognition output from the model +class Recognition { + /// Index of the result + final int _id; + /// Label of the result + final String _label; + /// Confidence [0.0, 1.0] + final double _score; + + Recognition(this._id, this._label, this._score); + + int get id => _id; + String get label => _label; + double get score => _score; + + @override + String toString() { + return 'Recognition(id: $id, label: $label, score: $score)'; + } +} diff --git a/lib/utils/stats.dart b/lib/utils/stats.dart new file mode 100644 index 0000000..f1582f6 --- /dev/null +++ b/lib/utils/stats.dart @@ -0,0 +1,23 @@ +/// Bundles different elapsed times +class Stats { + /// Total time taken in the isolate where the inference runs + int totalPredictTime; + + /// [totalPredictTime] + communication overhead time + /// between main isolate and another isolate + int totalElapsedTime; + + /// Time for which inference runs + int inferenceTime; + + /// Time taken to pre-process the image + int preProcessingTime; + + Stats(this.totalPredictTime, this.totalElapsedTime, this.inferenceTime, + this.preProcessingTime); + + @override + String toString() { + return 'Stats{totalPredictTime: $totalPredictTime, totalElapsedTime: $totalElapsedTime, inferenceTime: $inferenceTime, preProcessingTime: $preProcessingTime}'; + } +} diff --git a/pubspec.lock b/pubspec.lock index 51bab1f..12fa155 100644 --- a/pubspec.lock +++ b/pubspec.lock @@ -1,6 +1,13 @@ # Generated by pub # See https://dart.dev/tools/pub/glossary#lockfile packages: + archive: + dependency: transitive + description: + name: archive + url: "https://pub.dartlang.org" + source: hosted + version: "3.3.0" async: dependency: transitive description: @@ -71,6 +78,13 @@ packages: url: "https://pub.dartlang.org" source: hosted version: "0.3.3+1" + crypto: + dependency: transitive + description: + name: crypto + url: "https://pub.dartlang.org" + source: hosted + version: "3.0.2" cupertino_icons: dependency: "direct main" description: @@ -92,6 +106,13 @@ packages: url: "https://pub.dartlang.org" source: hosted version: "1.2.1" + file: + dependency: transitive + description: + name: file + url: "https://pub.dartlang.org" + source: hosted + version: "6.1.2" flutter: dependency: "direct main" description: flutter @@ -121,6 +142,13 @@ packages: description: flutter source: sdk version: "0.0.0" + image: + dependency: "direct main" + description: + name: image + url: "https://pub.dartlang.org" + source: hosted + version: "3.2.0" js: dependency: transitive description: @@ -170,6 +198,69 @@ packages: url: "https://pub.dartlang.org" source: hosted version: "1.8.1" + path_provider: + dependency: "direct main" + description: + name: path_provider + url: "https://pub.dartlang.org" + source: hosted + version: "2.0.11" + path_provider_android: + dependency: transitive + description: + name: path_provider_android + url: "https://pub.dartlang.org" + source: hosted + version: "2.0.15" + path_provider_ios: + dependency: transitive + description: + name: path_provider_ios + url: "https://pub.dartlang.org" + source: hosted + version: "2.0.10" + path_provider_linux: + dependency: transitive + description: + name: path_provider_linux + url: "https://pub.dartlang.org" + source: hosted + version: "2.1.7" + path_provider_macos: + dependency: transitive + description: + name: path_provider_macos + url: "https://pub.dartlang.org" + source: hosted + version: "2.0.6" + path_provider_platform_interface: + dependency: transitive + description: + name: path_provider_platform_interface + url: "https://pub.dartlang.org" + source: hosted + version: "2.0.4" + path_provider_windows: + dependency: transitive + description: + name: path_provider_windows + url: "https://pub.dartlang.org" + source: hosted + version: "2.0.7" + petitparser: + dependency: transitive + description: + name: petitparser + url: "https://pub.dartlang.org" + source: hosted + version: "5.0.0" + platform: + dependency: transitive + description: + name: platform + url: "https://pub.dartlang.org" + source: hosted + version: "3.1.0" plugin_platform_interface: dependency: transitive description: @@ -177,6 +268,13 @@ packages: url: "https://pub.dartlang.org" source: hosted version: "2.1.2" + process: + dependency: transitive + description: + name: process + url: "https://pub.dartlang.org" + source: hosted + version: "4.2.4" quiver: dependency: transitive description: @@ -245,6 +343,13 @@ packages: url: "https://pub.dartlang.org" source: hosted version: "0.9.0" + typed_data: + dependency: transitive + description: + name: typed_data + url: "https://pub.dartlang.org" + source: hosted + version: "1.3.1" vector_math: dependency: transitive description: @@ -252,6 +357,27 @@ packages: url: "https://pub.dartlang.org" source: hosted version: "2.1.2" + win32: + dependency: transitive + description: + name: win32 + url: "https://pub.dartlang.org" + source: hosted + version: "2.6.1" + xdg_directories: + dependency: transitive + description: + name: xdg_directories + url: "https://pub.dartlang.org" + source: hosted + version: "0.2.0+1" + xml: + dependency: transitive + description: + name: xml + url: "https://pub.dartlang.org" + source: hosted + version: "6.1.0" sdks: dart: ">=2.17.1 <3.0.0" - flutter: ">=2.8.0" + flutter: ">=3.0.0" diff --git a/pubspec.yaml b/pubspec.yaml index a04a5c7..6cdd924 100644 --- a/pubspec.yaml +++ b/pubspec.yaml @@ -34,9 +34,11 @@ dependencies: # The following adds the Cupertino Icons font to your application. # Use with the CupertinoIcons class for iOS style icons. cupertino_icons: ^1.0.2 - tflite_flutter: ^0.9.0 - logger: ^1.1.0 camera: ^0.9.7+1 + image: ^3.2.0 + logger: ^1.1.0 + path_provider: ^2.0.11 + tflite_flutter: ^0.9.0 dev_dependencies: flutter_test: @@ -61,7 +63,8 @@ flutter: uses-material-design: true # To add assets to your application, add an assets section, like this: - # assets: + assets: + - assets/ # - images/a_dot_burr.jpeg # - images/a_dot_ham.jpeg