Experiments

Get User Voice – Full code

This is the code used in the final version of Get User Voice that was presented as part of Tentacular Voice: A Solo Exhibition. Utilising JavaScript to create an interactivity between the frequency value from a computer’s internal microphone and the rotation of a .obj file. Also incorporated is an exported layered animation from Blender that is triggered using the keyboard keys F, G and H.

<!DOCTYPE html>
<html>
<head>
<title>Animation Reaction to Audio Frequency of Mic Audio 1</title>

<style>
body{margin: 0;padding: 0; background: #000;}
canvas{display: block;}
a{
color: #0080ff;
}

.info{
color: #fff;
text-align: center;
position: absolute;
top: 0;
box-sizing: border-box;
width: 100%;
padding: 5px;
}
</style>

<script src=”three.js-master/build/three.js”></script>

<script src=”three.js-master/examples/js/loaders/OBJLoader.js”></script>

</head>

<body>

<script>

‘use strict’;

document.addEventListener(‘keydown’, (event) => {
const keyName = event.key;
if (keyName === ‘f’) {
fadeAction( ‘idle’ );
}
if (keyName === ‘g’) {
fadeAction( ‘run’ );
}
if (keyName === ‘h’) {
fadeAction( ‘wrap’ );
}
});

var width, height, clock, scene, camera, renderer;
var loader = new THREE.JSONLoader();
var ambientLight, mesh, action = {}, mixer, fadeAction;
var mouthone;
var mouthtwo;
var array_freq_domain;
var audioContext = new AudioContext();
var BUFF_SIZE_RENDERER = 16384;
var audioInput = null,
microphone_stream = null,
gain_node = null,
script_processor_node = null,
script_processor_analysis_node = null,
analyser_node = null;

clock  = new THREE.Clock();

//////////////////////////
//  Mic set-up                     //
//////////////////////////

if (!navigator.getUserMedia)
navigator.getUserMedia = navigator.getUserMedia || navigator.webkitGetUserMedia ||
navigator.mozGetUserMedia || navigator.msGetUserMedia;

if (navigator.getUserMedia){
navigator.getUserMedia({audio:true},
function(stream) {
start_microphone(stream);
},

function(e) {
alert(‘Error capturing audio.’);
}
);

} else {alert(‘getUserMedia not supported in this browser.’);}
function process_microphone_buffer(event) {
var i, N, inp, microphone_output_buffer;
microphone_output_buffer = event.inputBuffer.getChannelData(0);
}

//////////////////////////
//  3D                                   //
//////////////////////////

var scene = new THREE.Scene();
var camera = new THREE.PerspectiveCamera( 75, window.innerWidth/window.innerHeight, 0.1, 1000 );
var ambient = new THREE.AmbientLight( 0x101030 );
scene.add( ambient );
var directionalLight = new THREE.DirectionalLight( 0xffeedd );
directionalLight.position.set( 1, 2, 1 );
scene.add( directionalLight );

var renderer = new THREE.WebGLRenderer();
renderer.setSize( window.innerWidth, window.innerHeight );
renderer.setClearColor(0xBDBDBD, 1);
document.body.appendChild( renderer.domElement );
var material = new THREE.MeshBasicMaterial( {color: 0x00ff00} );
var texture = new THREE.ImageUtils.loadTexture( “mouth3d8_smooth.jpg” );
var material = new THREE.MeshLambertMaterial( {map: texture} );

camera.position.z = 8;

var loaderone = new THREE.OBJLoader();
loaderone.load( ‘mouthtop2.obj’, function ( objectone ) {
objectone.traverse( function ( child ) {
if ( child instanceof THREE.Mesh ) {
child.material = material;
mouthone = objectone;
mouthone.position.y = 0.2;
mouthone.position.x = -4;
mouthone.position.z = 1.3;
scene.add( mouthone );
}
} );
}
);

var loadertwo = new THREE.OBJLoader();
loadertwo.load( ‘mouthbottom4.obj’, function ( objecttwo ) {
objecttwo.traverse( function ( child ) {
if ( child instanceof THREE.Mesh ) {
child.material = material;
mouthtwo = objecttwo;
mouthtwo.position.y = 0.2;
mouthtwo.position.x = -4;
mouthtwo.position.z = 1.3;
scene.add( mouthtwo );
}
} );
}
);

////////////////////////
// Animation loader  //
//////////////////////

loader.load( ‘tentacleanimation6.json’, function( geometry, materials ) {
materials.forEach( function ( material ) {
material.skinning = true;
} );

mesh = new THREE.SkinnedMesh(
geometry,
new THREE.MeshFaceMaterial( materials )
);

mixer = new THREE.AnimationMixer( mesh );

action.idle = mixer.clipAction( geometry.animations[ 1 ] );
action.run = mixer.clipAction( geometry.animations[ 0 ] );
action.wrap = mixer.clipAction( geometry.animations[ 2 ] );
action.idle.setEffectiveWeight( 1 );
action.run.setEffectiveWeight( 1 );
action.wrap.setEffectiveWeight( 1 );
action.idle.play();
mesh.position.y = 0;
mesh.position.x = -3;
scene.add( mesh );
} );

fadeAction = function () {
var activeActionName = ‘idle’;
return function ( name ) {
var from = action[ activeActionName ].play();
var to = action[ name ].play();
from.enabled = true;
to.enabled = true;
from.crossFadeTo( to, .3 );
activeActionName = name;
}
}();

////////////////////////////
//   mouths react to mic       //
////////////////////////////

function start_microphone(stream){
gain_node = audioContext.createGain();
gain_node.connect( audioContext.destination );
microphone_stream = audioContext.createMediaStreamSource(stream);
microphone_stream.connect(gain_node);
script_processor_node = audioContext.createScriptProcessor(BUFF_SIZE_RENDERER, 1, 1);
script_processor_node.onaudioprocess = process_microphone_buffer;
microphone_stream.connect(script_processor_node);
script_processor_analysis_node = audioContext.createScriptProcessor(2048, 1, 1);
script_processor_analysis_node.connect(gain_node);
analyser_node = audioContext.createAnalyser();
analyser_node.smoothingTimeConstant = 0;
analyser_node.fftSize = 2048;
microphone_stream.connect(analyser_node);
analyser_node.connect(script_processor_analysis_node);
var buffer_length = analyser_node.frequencyBinCount;
array_freq_domain = new Uint8Array(buffer_length);

////////////////////////////////
// Assigning reaction to mic     //
//////////////////////////////

script_processor_analysis_node.onaudioprocess = function() {

// get the average for the first channel
analyser_node.getByteFrequencyData(array_freq_domain);

// animate the mouth
if (microphone_stream.playbackState == microphone_stream.PLAYING_STATE) {
var render = function () {
for (var i = 0; i < (array_freq_domain.length); i++){
var valueone = array_freq_domain[array_freq_domain.length / 2];
mouthone.rotation.x = deg2rad( valueone ) / 7;
var valuetwo = array_freq_domain[6];
mouthtwo.rotation.x = deg2rad( valuetwo ) / 7;
}

renderer.render(scene, camera);
}

requestAnimationFrame( render );

function deg2rad( deg ){
return deg * Math.PI / 180;
}

}
}
};

;( function update () {
requestAnimationFrame( update );
var delta = clock.getDelta();
var theta = clock.getElapsedTime();
if ( mixer ) { mixer.update( delta ); }
renderer.render( scene, camera );
} )();

</script>
</body>
</html>

Standard

Leave a Reply

Your email address will not be published. Required fields are marked *