init memos char rnn example

example_frozen_graph_char_rnn/LICENSE

+Copyright 2015 The TensorFlow Authors.  All rights reserved.
+
+                                 Apache License
+                           Version 2.0, January 2004
+                        http://www.apache.org/licenses/
+
+   TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
+
+   1. Definitions.
+
+      "License" shall mean the terms and conditions for use, reproduction,
+      and distribution as defined by Sections 1 through 9 of this document.
+
+      "Licensor" shall mean the copyright owner or entity authorized by
+      the copyright owner that is granting the License.
+
+      "Legal Entity" shall mean the union of the acting entity and all
+      other entities that control, are controlled by, or are under common
+      control with that entity. For the purposes of this definition,
+      "control" means (i) the power, direct or indirect, to cause the
+      direction or management of such entity, whether by contract or
+      otherwise, or (ii) ownership of fifty percent (50%) or more of the
+      outstanding shares, or (iii) beneficial ownership of such entity.
+
+      "You" (or "Your") shall mean an individual or Legal Entity
+      exercising permissions granted by this License.
+
+      "Source" form shall mean the preferred form for making modifications,
+      including but not limited to software source code, documentation
+      source, and configuration files.
+
+      "Object" form shall mean any form resulting from mechanical
+      transformation or translation of a Source form, including but
+      not limited to compiled object code, generated documentation,
+      and conversions to other media types.
+
+      "Work" shall mean the work of authorship, whether in Source or
+      Object form, made available under the License, as indicated by a
+      copyright notice that is included in or attached to the work
+      (an example is provided in the Appendix below).
+
+      "Derivative Works" shall mean any work, whether in Source or Object
+      form, that is based on (or derived from) the Work and for which the
+      editorial revisions, annotations, elaborations, or other modifications
+      represent, as a whole, an original work of authorship. For the purposes
+      of this License, Derivative Works shall not include works that remain
+      separable from, or merely link (or bind by name) to the interfaces of,
+      the Work and Derivative Works thereof.
+
+      "Contribution" shall mean any work of authorship, including
+      the original version of the Work and any modifications or additions
+      to that Work or Derivative Works thereof, that is intentionally
+      submitted to Licensor for inclusion in the Work by the copyright owner
+      or by an individual or Legal Entity authorized to submit on behalf of
+      the copyright owner. For the purposes of this definition, "submitted"
+      means any form of electronic, verbal, or written communication sent
+      to the Licensor or its representatives, including but not limited to
+      communication on electronic mailing lists, source code control systems,
+      and issue tracking systems that are managed by, or on behalf of, the
+      Licensor for the purpose of discussing and improving the Work, but
+      excluding communication that is conspicuously marked or otherwise
+      designated in writing by the copyright owner as "Not a Contribution."
+
+      "Contributor" shall mean Licensor and any individual or Legal Entity
+      on behalf of whom a Contribution has been received by Licensor and
+      subsequently incorporated within the Work.
+
+   2. Grant of Copyright License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      copyright license to reproduce, prepare Derivative Works of,
+      publicly display, publicly perform, sublicense, and distribute the
+      Work and such Derivative Works in Source or Object form.
+
+   3. Grant of Patent License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      (except as stated in this section) patent license to make, have made,
+      use, offer to sell, sell, import, and otherwise transfer the Work,
+      where such license applies only to those patent claims licensable
+      by such Contributor that are necessarily infringed by their
+      Contribution(s) alone or by combination of their Contribution(s)
+      with the Work to which such Contribution(s) was submitted. If You
+      institute patent litigation against any entity (including a
+      cross-claim or counterclaim in a lawsuit) alleging that the Work
+      or a Contribution incorporated within the Work constitutes direct
+      or contributory patent infringement, then any patent licenses
+      granted to You under this License for that Work shall terminate
+      as of the date such litigation is filed.
+
+   4. Redistribution. You may reproduce and distribute copies of the
+      Work or Derivative Works thereof in any medium, with or without
+      modifications, and in Source or Object form, provided that You
+      meet the following conditions:
+
+      (a) You must give any other recipients of the Work or
+          Derivative Works a copy of this License; and
+
+      (b) You must cause any modified files to carry prominent notices
+          stating that You changed the files; and
+
+      (c) You must retain, in the Source form of any Derivative Works
+          that You distribute, all copyright, patent, trademark, and
+          attribution notices from the Source form of the Work,
+          excluding those notices that do not pertain to any part of
+          the Derivative Works; and
+
+      (d) If the Work includes a "NOTICE" text file as part of its
+          distribution, then any Derivative Works that You distribute must
+          include a readable copy of the attribution notices contained
+          within such NOTICE file, excluding those notices that do not
+          pertain to any part of the Derivative Works, in at least one
+          of the following places: within a NOTICE text file distributed
+          as part of the Derivative Works; within the Source form or
+          documentation, if provided along with the Derivative Works; or,
+          within a display generated by the Derivative Works, if and
+          wherever such third-party notices normally appear. The contents
+          of the NOTICE file are for informational purposes only and
+          do not modify the License. You may add Your own attribution
+          notices within Derivative Works that You distribute, alongside
+          or as an addendum to the NOTICE text from the Work, provided
+          that such additional attribution notices cannot be construed
+          as modifying the License.
+
+      You may add Your own copyright statement to Your modifications and
+      may provide additional or different license terms and conditions
+      for use, reproduction, or distribution of Your modifications, or
+      for any such Derivative Works as a whole, provided Your use,
+      reproduction, and distribution of the Work otherwise complies with
+      the conditions stated in this License.
+
+   5. Submission of Contributions. Unless You explicitly state otherwise,
+      any Contribution intentionally submitted for inclusion in the Work
+      by You to the Licensor shall be under the terms and conditions of
+      this License, without any additional terms or conditions.
+      Notwithstanding the above, nothing herein shall supersede or modify
+      the terms of any separate license agreement you may have executed
+      with Licensor regarding such Contributions.
+
+   6. Trademarks. This License does not grant permission to use the trade
+      names, trademarks, service marks, or product names of the Licensor,
+      except as required for reasonable and customary use in describing the
+      origin of the Work and reproducing the content of the NOTICE file.
+
+   7. Disclaimer of Warranty. Unless required by applicable law or
+      agreed to in writing, Licensor provides the Work (and each
+      Contributor provides its Contributions) on an "AS IS" BASIS,
+      WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+      implied, including, without limitation, any warranties or conditions
+      of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
+      PARTICULAR PURPOSE. You are solely responsible for determining the
+      appropriateness of using or redistributing the Work and assume any
+      risks associated with Your exercise of permissions under this License.
+
+   8. Limitation of Liability. In no event and under no legal theory,
+      whether in tort (including negligence), contract, or otherwise,
+      unless required by applicable law (such as deliberate and grossly
+      negligent acts) or agreed to in writing, shall any Contributor be
+      liable to You for damages, including any direct, indirect, special,
+      incidental, or consequential damages of any character arising as a
+      result of this License or out of the use or inability to use the
+      Work (including but not limited to damages for loss of goodwill,
+      work stoppage, computer failure or malfunction, or any and all
+      other commercial damages or losses), even if such Contributor
+      has been advised of the possibility of such damages.
+
+   9. Accepting Warranty or Additional Liability. While redistributing
+      the Work or Derivative Works thereof, You may choose to offer,
+      and charge a fee for, acceptance of support, warranty, indemnity,
+      or other liability obligations and/or rights consistent with this
+      License. However, in accepting such obligations, You may act only
+      on Your own behalf and on Your sole responsibility, not on behalf
+      of any other Contributor, and only if You agree to indemnify,
+      defend, and hold each Contributor harmless for any liability
+      incurred by, or claims asserted against, such Contributor by reason
+      of your accepting any such warranty or additional liability.
+
+   END OF TERMS AND CONDITIONS
+
+   APPENDIX: How to apply the Apache License to your work.
+
+      To apply the Apache License to your work, attach the following
+      boilerplate notice, with the fields enclosed by brackets "[]"
+      replaced with your own identifying information. (Don't include
+      the brackets!)  The text should be enclosed in the appropriate
+      comment syntax for the file format. We also recommend that a
+      file or class name and description of purpose be included on the
+      same "printed page" as the copyright notice for easier
+      identification within third-party archives.
+
+   Copyright 2015, The TensorFlow Authors.
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.
\ No newline at end of file

example_frozen_graph_char_rnn/Makefile

+# Attempt to load a config.make file.
+# If none is found, project defaults in config.project.make will be used.
+ifneq ($(wildcard config.make),)
+	include config.make
+endif
+
+# make sure the the OF_ROOT location is defined
+ifndef OF_ROOT
+	OF_ROOT=$(realpath ../../..)
+endif
+
+# call the project makefile!
+include $(OF_ROOT)/libs/openFrameworksCompiled/project/makefileCommon/compile.project.mk
+
+# ofxTensorFlow2
+include $(OF_ROOT)/addons/ofxTensorFlow2/addon_targets.mk

example_frozen_graph_char_rnn/README.md

+# Basic example
+This example demonstrates how to load and run a frozen graph developed in TensorFlow 1.
+
+### TensorFlow2
+
+### openFrameworks
+By default `ofxTF2::Model` will use the `SavedModel` format. To use the `FrozenGraph` format you may either add the type to the constructor or call `setModelType()` afterwards.
+As the default names differ from the names in the `SavedModel` format make sure to overwrite names of the ins and outs by calling `setup()`.
+```c++
+// set model type and i/o names
+model.setModelType(cppflow::model::TYPE::FROZEN_GRAPH);
+model.setup({{"x:0"}}, {{"Identity:0"}});
+```
+Afterwards you can load the _pb file_ using the `load()` function.
+```c++
+// load the model, bail out on error
+if(!model.load("model.pb")) {
+	std::exit(EXIT_FAILURE);
+}
+```
+
+Everything else should work the same.
+
+Please understand that we wont be able to invest a lot of time in supporting this feature in the future.
\ No newline at end of file

example_frozen_graph_char_rnn/addons.make

+ofxTensorFlow2

example_frozen_graph_char_rnn/config.make

+################################################################################
+# CONFIGURE PROJECT MAKEFILE (optional)
+#   This file is where we make project specific configurations.
+################################################################################
+
+################################################################################
+# OF ROOT
+#   The location of your root openFrameworks installation
+#       (default) OF_ROOT = ../../.. 
+################################################################################
+# OF_ROOT = ../../..
+
+################################################################################
+# PROJECT ROOT
+#   The location of the project - a starting place for searching for files
+#       (default) PROJECT_ROOT = . (this directory)
+#    
+################################################################################
+# PROJECT_ROOT = .
+
+################################################################################
+# PROJECT SPECIFIC CHECKS
+#   This is a project defined section to create internal makefile flags to 
+#   conditionally enable or disable the addition of various features within 
+#   this makefile.  For instance, if you want to make changes based on whether
+#   GTK is installed, one might test that here and create a variable to check. 
+################################################################################
+# None
+
+################################################################################
+# PROJECT EXTERNAL SOURCE PATHS
+#   These are fully qualified paths that are not within the PROJECT_ROOT folder.
+#   Like source folders in the PROJECT_ROOT, these paths are subject to 
+#   exlclusion via the PROJECT_EXLCUSIONS list.
+#
+#     (default) PROJECT_EXTERNAL_SOURCE_PATHS = (blank) 
+#
+#   Note: Leave a leading space when adding list items with the += operator
+################################################################################
+# PROJECT_EXTERNAL_SOURCE_PATHS = 
+
+################################################################################
+# PROJECT EXCLUSIONS
+#   These makefiles assume that all folders in your current project directory 
+#   and any listed in the PROJECT_EXTERNAL_SOURCH_PATHS are are valid locations
+#   to look for source code. The any folders or files that match any of the 
+#   items in the PROJECT_EXCLUSIONS list below will be ignored.
+#
+#   Each item in the PROJECT_EXCLUSIONS list will be treated as a complete 
+#   string unless teh user adds a wildcard (%) operator to match subdirectories.
+#   GNU make only allows one wildcard for matching.  The second wildcard (%) is
+#   treated literally.
+#
+#      (default) PROJECT_EXCLUSIONS = (blank)
+#
+#		Will automatically exclude the following:
+#
+#			$(PROJECT_ROOT)/bin%
+#			$(PROJECT_ROOT)/obj%
+#			$(PROJECT_ROOT)/%.xcodeproj
+#
+#   Note: Leave a leading space when adding list items with the += operator
+################################################################################
+# PROJECT_EXCLUSIONS =
+
+################################################################################
+# PROJECT LINKER FLAGS
+#	These flags will be sent to the linker when compiling the executable.
+#
+#		(default) PROJECT_LDFLAGS = -Wl,-rpath=./libs
+#
+#   Note: Leave a leading space when adding list items with the += operator
+#
+# Currently, shared libraries that are needed are copied to the 
+# $(PROJECT_ROOT)/bin/libs directory.  The following LDFLAGS tell the linker to
+# add a runtime path to search for those shared libraries, since they aren't 
+# incorporated directly into the final executable application binary.
+################################################################################
+# PROJECT_LDFLAGS=-Wl,-rpath=./libs
+
+################################################################################
+# PROJECT DEFINES
+#   Create a space-delimited list of DEFINES. The list will be converted into 
+#   CFLAGS with the "-D" flag later in the makefile.
+#
+#		(default) PROJECT_DEFINES = (blank)
+#
+#   Note: Leave a leading space when adding list items with the += operator
+################################################################################
+# PROJECT_DEFINES = 
+
+################################################################################
+# PROJECT CFLAGS
+#   This is a list of fully qualified CFLAGS required when compiling for this 
+#   project.  These CFLAGS will be used IN ADDITION TO the PLATFORM_CFLAGS 
+#   defined in your platform specific core configuration files. These flags are
+#   presented to the compiler BEFORE the PROJECT_OPTIMIZATION_CFLAGS below. 
+#
+#		(default) PROJECT_CFLAGS = (blank)
+#
+#   Note: Before adding PROJECT_CFLAGS, note that the PLATFORM_CFLAGS defined in 
+#   your platform specific configuration file will be applied by default and 
+#   further flags here may not be needed.
+#
+#   Note: Leave a leading space when adding list items with the += operator
+################################################################################
+# PROJECT_CFLAGS = 
+
+################################################################################
+# PROJECT OPTIMIZATION CFLAGS
+#   These are lists of CFLAGS that are target-specific.  While any flags could 
+#   be conditionally added, they are usually limited to optimization flags. 
+#   These flags are added BEFORE the PROJECT_CFLAGS.
+#
+#   PROJECT_OPTIMIZATION_CFLAGS_RELEASE flags are only applied to RELEASE targets.
+#
+#		(default) PROJECT_OPTIMIZATION_CFLAGS_RELEASE = (blank)
+#
+#   PROJECT_OPTIMIZATION_CFLAGS_DEBUG flags are only applied to DEBUG targets.
+#
+#		(default) PROJECT_OPTIMIZATION_CFLAGS_DEBUG = (blank)
+#
+#   Note: Before adding PROJECT_OPTIMIZATION_CFLAGS, please note that the 
+#   PLATFORM_OPTIMIZATION_CFLAGS defined in your platform specific configuration 
+#   file will be applied by default and further optimization flags here may not 
+#   be needed.
+#
+#   Note: Leave a leading space when adding list items with the += operator
+################################################################################
+# PROJECT_OPTIMIZATION_CFLAGS_RELEASE = 
+# PROJECT_OPTIMIZATION_CFLAGS_DEBUG = 
+
+################################################################################
+# PROJECT COMPILERS
+#   Custom compilers can be set for CC and CXX
+#		(default) PROJECT_CXX = (blank)
+#		(default) PROJECT_CC = (blank)
+#   Note: Leave a leading space when adding list items with the += operator
+################################################################################
+# PROJECT_CXX = 
+# PROJECT_CC = 

example_frozen_graph_char_rnn/src/main.cpp

+#include "ofMain.h"
+#include "ofApp.h"
+
+//========================================================================
+int main() {
+
+	ofSetupOpenGL(1024, 768, OF_WINDOW); // <-------- setup the GL context
+
+	// this kicks off the running of my app
+	// can be OF_WINDOW or OF_FULLSCREEN
+	// pass in width and height too:
+	ofRunApp(new ofApp());
+}

example_frozen_graph_char_rnn/src/ofApp.cpp

+/*
+ * ofxTensorFlow2
+ *
+ * Copyright (c) 2021 ZKM | Hertz-Lab
+ * Paul Bethge <bethge@zkm.de>
+ * Dan Wilcox <dan.wilcox@zkm.de>
+ *
+ * BSD Simplified License.
+ * For information on usage and redistribution, and for a DISCLAIMER OF ALL
+ * WARRANTIES, see the file, "LICENSE.txt," in this distribution.
+ *
+ * This code has been developed at ZKM | Hertz-Lab as part of „The Intelligent 
+ * Museum“ generously funded by the German Federal Cultural Foundation.
+ * 
+ * This code is based on Memo Akten's ofxMSATensorFlow example.
+ */
+
+#include "ofApp.h"
+
+//--------------------------------------------------------------
+void ofApp::setup() {
+	ofSetColor(255);
+	ofBackground(0);
+	ofSetVerticalSync(true);
+	ofSetLogLevel(OF_LOG_VERBOSE);
+	ofSetWindowTitle("example_frozen_graph_char_rnn");
+	ofSetFrameRate(20); // generating a character per frame at 60fps is too fast to read in realtime
+
+	// set model type and i/o names
+	model.setModelType(cppflow::model::TYPE::FROZEN_GRAPH);
+	std::vector<std::string> inputNames = {
+		"data_in",
+		"state_in",
+	};
+	std::vector<std::string> outputNames = {
+		"data_out",
+		"state_out",
+	};
+	model.setup(inputNames, outputNames);
+
+
+	// scan models dir
+	models_dir.listDir("models");
+	if(models_dir.size()==0) {
+		ofLogError() << "Couldn't find models folder.";
+		assert(false);
+		ofExit(1);
+	}
+	models_dir.sort();
+	load_model_index(0); // load first model
+
+	// seed rng
+	rng.seed(ofGetSystemTimeMicros());
+	
+	// load a font for displaying strings
+	font.load(OF_TTF_SANS, 14);
+}
+
+
+//--------------------------------------------------------------
+// Load model by folder INDEX
+void ofApp::load_model_index(int index) {
+	cur_model_index = ofClamp(index, 0, models_dir.size()-1);
+	load_model(models_dir.getPath(cur_model_index));
+}
+
+
+//--------------------------------------------------------------
+// Load graph (model trained in and exported from python) by folder NAME, and initialize session
+void ofApp::load_model(std::string dir) {
+
+	// TODO load model from 'dir'
+	// load the model, bail out on error
+	const std::string model_path = dir + "/graph_frz.pb";
+	if(!model.load(model_path)) {
+		std::exit(EXIT_FAILURE);
+	}
+
+	// load character map
+	// TODO load model from 'dir'
+	const std::string chars_path = dir + "/chars.txt";
+	load_chars(chars_path);
+
+	// init tensor for input
+	// needs to be a single int (index of character)
+	// HOWEVER input is not a scalar or vector, but a rank 2 tensor with shape {1, 1} (i.e. a matrix)
+	// WHY? because that's how the model was designed to make the internal calculations easier (batch size etc)
+	// TBH the model could be redesigned to accept just a rank 1 scalar, and then internally reshaped, but I'm lazy
+	t_data_in = cppflow::fill({1, 1}, 1, TF_INT32);
+
+	// prime model
+	prime_model(text_full, prime_length);
+}
+
+
+//--------------------------------------------------------------
+// load character <-> index mapping
+void ofApp::load_chars(string path) {
+	ofLogVerbose() << "load_chars : " << path;
+	int_to_char.clear();
+	char_to_int.clear();
+	ofBuffer buffer = ofBufferFromFile(path);
+
+	for(auto line : buffer.getLines()) {
+		char c = ofToInt(line); // TODO: will this manage unicode?
+		int_to_char.push_back(c);
+		int i = int_to_char.size()-1;
+		char_to_int[c] = i;
+		ofLogVerbose() << i << " : " << c;
+	}
+}
+
+
+
+//--------------------------------------------------------------
+// prime model with a sequence of characters
+// this runs the data through the model element by element, so as to update its internal state (stored in t_state)
+// next time we feed the model an element to make a prediction, it will make the prediction primed on this state (i.e. sequence of elements)
+void ofApp::prime_model(string prime_data, int prime_length) {
+
+	ofLogVerbose() << "prime_model : " << prime_data << " (" << prime_length << ")";
+
+	outputReady = false;
+
+	for(int i=MAX(0, prime_data.size()-prime_length); i<prime_data.size(); i++) {
+		run_model(prime_data[i]);
+	}
+}
+
+
+template<typename T> vector<T> adjust_probs_with_temp(const vector<T>& p_in, float t) {
+    if(t>0) {
+        vector<T> p_out(p_in.size());
+        T sum = 0;
+        for(size_t i=0; i<p_in.size(); i++) {
+            p_out[i] = exp( log((double)p_in[i]) / (double)t );
+            sum += p_out[i];
+        }
+
+        if(sum > 0)
+            for(size_t i=0; i<p_out.size(); i++) p_out[i] /= sum;
+
+        return p_out;
+    }
+
+    return p_in;
+}
+
+//--------------------------------------------------------------
+// run model on a single character
+void ofApp::run_model(char ch) {
+
+	ofLogVerbose() << "run_model : " << ch << " (" << char_to_int[ch] << ")";
+
+
+	// copy input data into tensor
+	// ofxTF2::vectorToTensor<int32_t>(std::vector<int32_t>(char_to_int[ch]));
+	// t_data_in = {char_to_int[ch]};
+
+	t_data_in = cppflow::fill({1, 1}, (int)char_to_int[ch], TF_INT32);
+
+	std::vector<cppflow::tensor> vectorOfInputTensors = {t_data_in};
+	
+	if(outputReady) {
+		// use state_in if passed in as parameter
+		vectorOfInputTensors.push_back(t_state);
+		ofLogVerbose() << "state_in is not empty";
+	}
+	else {
+		ofLogVerbose() << "state_in is empty";
+	}
+
+	auto vectorOfOutputTensors = model.runMultiModel(vectorOfInputTensors);
+
+	// convert model output from tensors to more manageable types
+	if(vectorOfOutputTensors.size() > 1) {
+		ofxTF2::tensorToVector<float>(vectorOfOutputTensors[0], last_model_output);
+		last_model_output = adjust_probs_with_temp(last_model_output, sample_temp);
+
+		// save lstm state for next run
+		t_state = vectorOfOutputTensors[1];
+	}
+	outputReady = true;
+}
+
+
+//--------------------------------------------------------------
+// add character to string, manage ghetto wrapping for display, run model etc.
+void ofApp::add_char(char ch) {
+	// add sampled char to text
+	if(ch == '\n') {
+		text_lines.push_back("");
+	} else {
+		text_lines.back() += ch;
+	}
+
+	// ghetto word wrap
+	if(text_lines.back().size() > max_line_width) {
+		string text_line_cur = text_lines.back();
+		text_lines.pop_back();
+		auto last_word_pos = text_line_cur.find_last_of(" ");
+		text_lines.push_back(text_line_cur.substr(0, last_word_pos));
+		text_lines.push_back(text_line_cur.substr(last_word_pos));
+	}
+
+	// ghetto scroll
+	while(text_lines.size() > max_line_num) text_lines.pop_front();
+
+	// rebuild text
+	text_full.clear();
+	for(auto&& text_line : text_lines) {
+		text_full += "\n" + text_line;
+	}
+