Using socket io stream server as middleman to send streams from one client to another

[joshterrill]

trafficstars

Hello,

I am trying to use socket.io-stream to send video data from getUserMedia from one client (the source), into another client where it will stream it into a <video> container. I'm using nwjs.io (node-webkit) as my source. I'm able to get desktop sharing video via:

var gui = require("nw.gui");
gui.Screen.chooseDesktopMedia(["window","screen"], function(streamId) {
    var vid_constraint = {
        mandatory: {
            chromeMediaSource: 'desktop', 
            chromeMediaSourceId: streamId, 
            maxWidth: 1920, 
            maxHeight: 1080,
            minFrameRate: 1,
            maxFrameRate: 5
        }, 
        optional:[]
    };
    navigator.webkitGetUserMedia({audio:false,video: vid_constraint}, 
        function(stream) {
            document.getElementById('video_1').src = URL.createObjectURL(stream);
            stream.onended = function() { 
                console.log("Ended"); 
            };
        }, 
        function(error) {
            console.log('failure',error);
        }
    );
});

This starts a screen share, and sends the stream into <video id="video_1"></video>. What I would like to do, is use socket.io-stream to take this video data, and send it to another client, probably a chrome browser. I've tried a few things...

var socket = io.connect("http://localhost:3000");
navigator.webkitGetUserMedia({audio:false,video: vid_constraint}, 
        function(stream) {
            //document.getElementById('video_1').src = URL.createObjectURL(stream);
                        ss(socket).emit("stream", stream);
            stream.onended = function() { 
                console.log("Ended"); 
            };
        }, 
        function(error) {
            console.log('failure',error);
        }
    );

And then on NodeJS server:

var express = require("express");
var app = express();
var http = require('http').Server(app);
var io = require('socket.io')(http);
var ss = require('socket.io-stream');

app.use(express.static('views'));

app.get('/', function(req, res){
  res.sendfile('views/index.html');
});

io.on('connection', function(socket){
  ss(socket).on('stream', function(stream) {
    ss(socket).emit("received-stream", stream);
  });
});

http.listen(3000, function(){
  console.log('listening on *:3000');
});

And then on the other client, the views/index.html file being served by the nodejs server:

var socket = io.connect("http://localhost:3000");
ss(socket).on("received-stream", function(data) {
  console.log(data);
  document.getElementById('video_1').src = URL.createObjectURL(data);
})

I can't seem to get both of the client's to talk to each other....

[BipinBhandari]

I am also facing the same problem. @joshterrill have you figured out the solution yet?

[joshterrill]

@BipinBhandari I have not. I kind of just gave up on it, honestly :(

[mmmmoj]

There is a different in data types, getUserMedia's stream is not same as socket's stream. so they will not work together without some modifications.

[molavy]

what modifications, sample code please?

[dmr07]

+1

[Premier]

+1

[AlvinLicuntao]

I use ss.createStream(stream) to format Mediastream to socket stream, but server throw exception ( throw new Error('stream has already been sent.');) ,Why ?

[ArturoJAT]

Any progress on this topic?, I am having the same issue.

[foxhound87]

+1

[Dcros]

Solution = encode the stream into a unit8 array stream with format .png and then in the other side recive it and pass it from base64 to img frames replayed on some : example: i use opencv in this example to encode the streamed frames from my webcam-->

function sendToWeb(err,pm) { if(err){ console.log("Cant Send the Frames")} pm = cv.imencode('.png',frameResized) socket.emit('frame', { pm });
};

this is how you encode the streamed video frames "frameResized" for getUserMedia(); i dont know how to encode but sure in google its something... {DONT TAKE IT LITTERALY ITS JUST A EXAMPLE FROM MY PROJECT only to get you the idea how to try it.}

you send the encoded unit8 array with .png format from server or client (in my case dosent matter works on both sides)

in the other side you should recive it as:

//image video frames buffer to Base64 function _arrayBufferToBase64(buffer) { var binary = ''; var bytes = new Uint8Array(buffer); var len = bytes.byteLength; for (var i = 0; i < len; i++) { binary += String.fromCharCode( bytes[ i ] ); } return binary; }

//the printed recived video function startScan(){ var sstr = "StartEye" //ignore this just the start for which socket to use openIfSocket(sstr); var canvass = document.getElementById('canvas-fDR'); var context = canvass.getContext('2d'); var img = new Image(); // show loading notice context.fillStyle = '#333'; context.fillText('Loading...', canvass.width/2-30, canvass.height/3); socket.on('frame', function (data) { //the data.im its the encoded unit8 frames or video stream from where are sending it using socket.io //here you conver to base64 the unit8 array by the function _arrayBufferToBase64. var b64encoded = btoa(_arrayBufferToBase64(data.im)); //console.log(b64encoded) //var base64String = data.im; img.onload = function () { context.drawImage(this, 0, 0, canvass.width, canvass.height); }; img.src = 'data:image/png;base64,' + b64encoded; }); };

hole code here : https://github.com/Dcros/NodeJs-AI-Live-Face-Recognition-Voice-Controlled

[tageecc]

@joshterrill It seems nothing to do with data types，just do server-side code like this：

io.of('/').on('connection', function (socket) {
    ss(socket).on('hot', function (stream, data) {
        const _stream = ss.createStream();
        ss(socket).emit('hot', _stream);
        stream.pipe(_stream);
    });
});

[Dcros]

@tageecc Hmmm nice one i have tested can you put an example for client side? i have tryed in simpled as josh shows but nothig then i have tryed this :

` const socket = io(this.state.endpoint); var stream = ss.createStream(); navigator.mediaDevices.getUserMedia({ audio: false, video: true }) .then(function(mediaStream) { var video = document.querySelector('#video'); video.src = window.URL.createObjectURL(mediaStream); ss(socket).emit("stream", stream, mediaStream); fs.createReadStream(mediaStream).pipe(stream); video.onloadedmetadata = function(e) { video.play();

		}
	}).catch(function(err) {
		if (err) alert('Cant Record Webcam' + err);
	});

`

but send to server a empty object-thing like this

` IOStream { [0] _readableState: [0] ReadableState { [0] objectMode: false, [0] highWaterMark: 16384, [0] buffer: BufferList { head: null, tail: null, length: 0 }, [0] length: 0, [0] pipes: null, [0] pipesCount: 0, [0] flowing: null, [0] ended: false, [0] endEmitted: false, [0] reading: false, [0] sync: true, [0] needReadable: false, [0] emittedReadable: false, [0] readableListening: false, [0] resumeScheduled: false, [0] destroyed: false, [0] defaultEncoding: 'utf8', [0] awaitDrain: 0, [0] readingMore: false, [0] decoder: null, [0] encoding: null }, [0] readable: true, [0] domain: null, [0] _events: [0] { end: [ [Object], [Function] ], [0] finish: [ [Function], [Object] ], [0] error: [ [Function: onerror], [Function] ], [0] unpipe: [Function: onunpipe], [0] drain: [Function], [0] close: { [Function: bound onceWrapper] listener: [Function: onclose] } }, [0] _eventsCount: 6, [0] _maxListeners: undefined, [0] _writableState: [0] WritableState { [0] objectMode: false, [0] highWaterMark: 16384, [0] finalCalled: false, [0] needDrain: false, [0] ending: false, [0] ended: false, [0] finished: false, [0] destroyed: false, [0] decodeStrings: true, [0] defaultEncoding: 'utf8', [0] length: 0, [0] writing: false, [0] corked: 0, [0] sync: true, [0] bufferProcessing: false, [0] onwrite: [Function: bound onwrite], [0] writecb: null, [0] writelen: 0, [0] bufferedRequest: null, [0] lastBufferedRequest: null, [0] pendingcb: 0, [0] prefinished: false, [0] errorEmitted: false, [0] bufferedRequestCount: 0, [0] corkedRequestsFree: [0] { next: null, [0] entry: null, [0] finish: [Function: bound onCorkedFinish] } }, [0] writable: true, [0] allowHalfOpen: false, [0] options: undefined, [0] id: 'b3a3d311-44b2-42eb-903c-b01c6a4392a8', [0] socket: null, [0] pushBuffer: [], [0] writeBuffer: [], [0] _readable: false, [0] _writable: false }

`

because in client side cant start recording for cant do the -->

fs.createReadStream(mediaStream).pipe(stream);

im sure i have mesd up in the client side any hand would be nice.

PD: the goal its the same send the video captured from getUserMedia() to server and can use it for something like cv.videocapture() of Opencv Js...

[ghost]

Same issue here.... Did it solved ???

[DanielOX]

@joshterrill It seems nothing to do with data types，just do server-side code like this：

io.of('/').on('connection', function (socket) {
    ss(socket).on('hot', function (stream, data) {
        const _stream = ss.createStream();
        ss(socket).emit('hot', _stream);
        stream.pipe(_stream);
    });
});

let me try this, it should work

[omaraflak]

This works. socket.io v3.0.5

server

const io = require('socket.io')(3000)
const ss = require('socket.io-stream')

io.on('connection', socket => {
    ss(socket).on('file', (filename, stream) => {
        for (const [_, client] of io.of('/').sockets) {
            if (client.id != socket.id) {
                const newStream = ss.createStream()
                ss(client).emit('file', filename, newStream)
                stream.pipe(newStream)
            }
        }
    })

    socket.emit('send_file')
})

console.log(`Listening on http://localhost:3000...`)

client

const fs = require('fs')
const io = require('socket.io-client')
const ss = require('socket.io-stream')
const socket = io('http://localhost:3000')

socket.on('connect', () => {
    socket.on('send_file', () => {
        const stream = ss.createStream()
        stream.on('end', () => console.log('file sent'))
        ss(socket).emit('file', 'ok.txt', stream)
        fs.createReadStream('./files/test.txt').pipe(stream)
    })

    ss(socket).on('file', (filename, stream) => {
        const filepath = `./files/${filename}`
        stream.pipe(fs.createWriteStream(filepath))
        stream.on('end', () => console.log(`file saved to ${filepath}`))
    })
})

Start the server, then two instances of the client. Make sure the path files/test.txt exists client side.

[neerajkr007]

@OmarAflak i know this might be a really silly question but how do i use const fs = require('fs') in client side it gives an error saying require dosent exist

[omaraflak]

@OmarAflak i know this might be a really silly question but how do i use const fs = require('fs') in client side it gives an error saying require dosent exist

My code works for NodeJs. I guess you're trying to run the client on a webpage. In that case (I'm not that much of a web dev), you'll need to figure out a way to save the data coming from the stream without using fs. fs is just used to save the data into a file locally.

[neerajkr007]

@OmarAflak im using nodejs aswell but const fs = require('fs') in client side it gives an error saying require dosent exist "require" works fine for the app.js

[omaraflak]

@OmarAflak im using nodejs aswell but const fs = require('fs') in client side it gives an error saying require dosent exist "require" works fine for the app.js

When you say "client" you mean another NodeJs app ? Or a javascript code served on a webpage ? Cause it won't work on browser.

[neerajkr007]

yea i meant a javascript code served on a webpage. so is there any possible way to send files from a client(a webpage) to another client (webpage) through a node server, say deployed on heroku ?

[neerajkr007]

@OmarAflak what I want to do is stream a local video from a client(again a webpage) to another through a node server is it possible ? currently I'm using simple-peer's data channels to send chunks(arrayBuffers) of video files to other users and it then loads the video on their pages. it works but I'm looking for better alternatives