Commit 26508615 authored by Dan Wilcox's avatar Dan Wilcox
Browse files

added basic OSC sending, added newer attrnn model, started renaming to Language Identifier

parent 7558af5d
Language Identification within OpenFrameworks
=============================================
Language Identifier
===================
Identification of chosen languages from 5s long audio snippets.
Identification of chosen languages from 5s long audio snippets and sent over OSC (Open Sound Control) messages.
This code base has been developed by [ZKM | Hertz-Lab](https://zkm.de/en/about-the-zkm/organization/hertz-lab) as part of the project [»The Intelligent Museum«](#the-intelligent-museum).
Please raise issues, ask questions, throw in ideas or submit code, as this repository is intended to be an open platform to collaboratively improve langugae identification.
......@@ -10,37 +10,58 @@ Copyright (c) 2021 ZKM | Karlsruhe.
BSD Simplified License.
##### Tested Platforms
- MacBook Pro 2017, macOS 10.15 & openFrameworks 0.11.2
- MacBook Pro 2018, macOS 11.3.1 & openFrameworks 0.11.2
Tested Platforms
----------------
* MacBook Pro 2017, macOS 10.15 & openFrameworks 0.11.2
* MacBook Pro 2018, macOS 11.3.1 & openFrameworks 0.11.2
Structure
---------
##### Structure
* src/: contains the C++ Code that interfaces with OpenFrameworks
* bin/: contains the SavedModel trained with TensorFlow2
* bin/: contains the SavedModels trained with TensorFlow2
Installation
------------
### Installation
As with all OpenFrameworks examples put this projekt to a folder where '../../../'
As with all OpenFrameworks examples put this project to a folder where '../../../'
is the root folder of your OpenFrameworks installation.
### Run this code
##### Compile
Run this code
-------------
### Compile
```shell
cd $ROOT_OF_THIS_PROJECT
make
```
##### Execute
### Execute
```
make RunRelease
```
### Further Reading
Repo for the training and other models coming soon...
### OSC Communication
Sends to:
* address: `localhost` ie. `127.0.0.1`
* port: `9999`
Message specification:
* **/detected status**: detection status
- status: float, boolean 1 found - 0 lost
* **/lang index name confidence**: detected language
- index: int, language map index
- name: string, language map name
- confidence: float, confidence percentage 0 - 100
### Contribute
Contributions are very welcome!
Please send an email to author@zkm.de
The Intelligent Museum
----------------------
### The Intelligent Museum
An artistic-curatorial field of experimentation for deep learning and visitor participation
The [ZKM | Center for Art and Media](https://zkm.de/en) and the [Deutsches Museum Nuremberg](https://www.deutsches-museum.de/en/nuernberg/information/) cooperate with the goal of implementing an AI-supported exhibition. Together with researchers and international artists, new AI-based works of art will be realized during the next four years (2020-2023). They will be embedded in the AI-supported exhibition in both houses. The Project „The Intelligent Museum“ is funded by the Digital Culture Programme of the [Kulturstiftung des Bundes](https://www.kulturstiftung-des-bundes.de/en) (German Federal Cultural Foundation).
......
/*
* ofxTensorFlow2
* Language Identifier
*
* Copyright (c) 2021 ZKM | Hertz-Lab
* Paul Bethge <bethge@zkm.de>
......
......@@ -3,11 +3,6 @@
//========================================================================
int main() {
ofSetupOpenGL(500, 260, OF_WINDOW); // <-------- setup the GL context
// this kicks off the running of my app
// can be OF_WINDOW or OF_FULLSCREEN
// pass in width and height too:
ofSetupOpenGL(500, 260, OF_WINDOW);
ofRunApp(new ofApp());
}
/*
* ofxTensorFlow2
* Language Identifier
*
* Copyright (c) 2021 ZKM | Hertz-Lab
* Paul Bethge <bethge@zkm.de>
......@@ -19,12 +19,13 @@
void ofApp::setup() {
ofSetFrameRate(60);
ofSetVerticalSync(true);
ofSetWindowTitle("ofxLID");
ofSetWindowTitle("Language Identifier");
ofSetCircleResolution(80);
ofBackground(54, 54, 54);
// load the model, bail out on error
if(!model.load("model_4lang")) {
//if(!model.load("model_4lang")) {
if(!model.load("model_attrnn")) {
std::exit(EXIT_FAILURE);
}
......@@ -75,6 +76,9 @@ void ofApp::setup() {
output = model.runModel(test);
ofLog() << "Setup done";
ofLog() << "============================";
// osc
sender.setup(host, port);
}
//--------------------------------------------------------------
......@@ -95,9 +99,15 @@ void ofApp::update() {
float prob;
model.classify(sampleBuffers, downsamplingFactor, argMax, prob);
// only display label when probabilty is high enough
// only send & display label when probabilty is high enough
if(prob >= minConfidence) {
displayLabel = labelsMap[argMax];
ofxOscMessage message;
message.setAddress("/lang");
message.addIntArg(argMax);
message.addStringArg(labelsMap[argMax]);
message.addFloatArg(prob * 100);
sender.sendMessage(message);
}
else {
displayLabel = " ";
......@@ -111,7 +121,22 @@ void ofApp::update() {
// release the trigger signal and emit enable
trigger = false;
enable = true;
// detection stopped
ofxOscMessage message;
message.setAddress("/detecting");
message.addIntArg(0);
sender.sendMessage(message);
}
if(recordingStarted) {
// detection started
ofxOscMessage message;
message.setAddress("/detecting");
message.addIntArg(1);
sender.sendMessage(message);
recordingStarted = false;
}
}
//--------------------------------------------------------------
......@@ -153,6 +178,14 @@ void ofApp::draw() {
ofPopStyle();
}
//--------------------------------------------------------------
void ofApp::exit() {
ofxOscMessage message;
message.setAddress("/detecting");
message.addIntArg(0);
sender.sendMessage(message);
}
//--------------------------------------------------------------
void ofApp::audioIn(ofSoundBuffer & input) {
......@@ -178,6 +211,7 @@ void ofApp::audioIn(ofSoundBuffer & input) {
recordingCounter = sampleBuffers.size();
// trigger recording in the next function call
recording = true;
recordingStarted = true;
}
// if we didnt just trigger
else {
......
/*
* ofxTensorFlow2
* * Language Identifier
*
* Copyright (c) 2021 ZKM | Hertz-Lab
* Paul Bethge <bethge@zkm.de>
......@@ -17,6 +17,7 @@
#include "ofMain.h"
#include "ofxTensorFlow2.h"
#include "ofxOsc.h"
#include "labels.h"
// AudioBufferFifo & AudioClassifier model defined in utils.h
......@@ -28,6 +29,7 @@ class ofApp : public ofBaseApp {
void setup();
void update();
void exit();
void draw();
void audioIn(ofSoundBuffer & input);
......@@ -84,4 +86,10 @@ class ofApp : public ofBaseApp {
bool trigger = false;
bool enable = true;
bool recording = true;
// osc
ofxOscSender sender;
int port = 9999;
std::string host = "localhost";
bool recordingStarted = false;
};
/*
* ofxTensorFlow2
* Language Identifier
*
* Copyright (c) 2021 ZKM | Hertz-Lab
* Paul Bethge <bethge@zkm.de>
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment