Livestream Thumbnails

[rolandstarke]

I would like to add thumbnails for my hls live stream.

On the server i have files like

stream.m3u8
segment0.ts
thumb0.jpg
segment1.ts
thumb1.jpg
segment2.ts
thumb2.jpg

So for every segment I have a thumbnail that I want to show. Would that be possible?

[tjenkinson]

Hey @rolandstarke thanks this is similar to what I was going to suggest.

You can probably use the PLAYBACK_FRAGMENT_LOADED instead of a timeout.

https://github.com/clappr/clappr/blob/1994866b7c9bd1703c4729a5cae423c0ab3be9e6/src/playbacks/hls/hls.js#L526

Also it would be better to only remove the thumbnails that have left the stream and add the new ones, instead of clearing all of them on each updates, as this might cause the ui to flicker.

[tjenkinson]

Ah actually Hls.Events.LEVEL_UPDATED is probably the event you wanted. This should only be fired when the playlist changes, so once for vod.

On 5 Feb 2018, at 18:36, Roland Starke <[email protected]mailto:[email protected]> wrote:

Hey @tjenkinsonhttps://github.com/tjenkinson thanks,

I tried with player.core.getCurrentPlayback().on(Clappr.Events.PLAYBACK_FRAGMENT_LOADED, console.infohttp://console.info)

the event is in the right direction. For livestreams it works well. but i also have a hls of a static 7h video. In this case the thumbnails won't need to update that often.

I could improve the performance with removing backdropHeight and limiting the thumbnails to 100-200 pictures.

//remove some thumbnails if there are very much, so we have a maximum of 200 thumbnails var useEachXThumbnails = Math.ceil(thumbnails.length / 200); thumbnails = thumbnails.filter(function(t, index) { return index % useEachXThumbnails === 0 });

Removing all thumbnails and adding them again seems to be okay now. (Idk if in an livestream, when old fragments drop, all fragment.start times change. in this case i would need to update them all anyway. So for now i am to lazy to write an logic that only removes old and adds new thumbnails.)

— You are receiving this because you were mentioned. Reply to this email directly, view it on GitHubhttps://github.com/tjenkinson/clappr-thumbnails-plugin/issues/86#issuecomment-363159833, or mute the threadhttps://github.com/notifications/unsubscribe-auth/ADG-WfN-lfXQsXF3UeRoKuAs5zkuahVNks5tRzwUgaJpZM4R4I-p.

[rolandstarke]

Thanks a lot. For my livestream and VOD it is working now.

<div id="video"></div>


<script src="https://cdn.jsdelivr.net/npm/[email protected]/dist/clappr.min.js"></script>
<script src="https://cdn.jsdelivr.net/npm/[email protected]/dist/clappr-thumbnails-plugin.js"></script>
<script>
/* global Clappr */
var ClapprThumbnailsGeneratorPlugin = Clappr.UICorePlugin.extend({

    thumbnails: [],
    isBusy: false,
    thumbnailsPlugin: null,

    markUnbusy: function () {
        this.isBusy = false;
    },

    bindEvents: function () {
        this.thumbnailsPlugin = this.core.plugins.filter(plugin => plugin.name === 'scrub-thumbnails')[0];
        if (!this.thumbnailsPlugin) {
            console.error('ClapprThumbnailsGeneratorPlugin requires ClapprThumbnailsPlugin');
            return;
        }
        this.listenTo(this.core, Clappr.Events.CORE_READY, this.bindPlaybackEvents);
    },

    bindPlaybackEvents: function () {
        var currentPlayback = this.core.getCurrentPlayback();
        if (!currentPlayback._hls) {
            console.error('ClapprThumbnailsGeneratorPlugin requires HLS Playback');
            return;
        }

        currentPlayback._hls.on('hlsLevelUpdated', this.updateThumbnails.bind(this));

        if (currentPlayback.levels && currentPlayback.levels.length > 0) {
            this.updateThumbnails();
        }
    },

    updateThumbnails: function () {
        console.log('updateThumbnails called');
        var that = this;
        var currentPlayback = that.core.getCurrentPlayback();
        var level = (currentPlayback.levels[0] || {}).level;
        if (!level || !level.details || !level.details.fragments || !level.details.fragments.map) return;

        //get thumbnail paths and times from fragments
        var newThumnails = level.details.fragments.map(function (fragment) {
            return {
                time: fragment.start,
                url: fragment.baseurl + '/../' + fragment.relurl
                    .replace('segment', 'thumb')
                    .replace('.ts', '.jpg'), // segment0.ts --> thumb0.jpg
            };
        });

        //limit the thumbnails to a maximum of 200 images
        var useEachXThumbnails = Math.ceil(newThumnails.length / 200);
        newThumnails = newThumnails.filter(function (t, index) {
            return index % useEachXThumbnails === 0;
        });

        //check if there is a change. else we can stop here
        if (that.thumbnails.length === newThumnails.length
            && that.thumbnails.every(function (t, i) { return t.time === newThumnails[i].time && t.url === newThumnails[i].url; })
        ) {
            return;
        }

        //if the thumbnail plugin is still busy loading the images from the last update stop here
        if (that.isBusy) return;
        that.isBusy = true;

        console.log('updating thumbnails');
        that.thumbnailsPlugin.removeThumbnail(that.thumbnails).then(function () {
            that.thumbnails = newThumnails;
            return that.thumbnailsPlugin.addThumbnail(that.thumbnails);
        }).catch(console.error).then(that.markUnbusy.bind(that));
    }

});
</script>
<script>
var player = new Clappr.Player({
    source: "/camera/live/stream.m3u8",
    parentId: "#video",
    autoPlay: true,
    mute: true,
    persistConfig: false, /* do not save anything in localStorage */
    plugins: {
        core: [ClapprThumbnailsPlugin, ClapprThumbnailsGeneratorPlugin],
    },
    scrubThumbnails: {
        spotlightHeight: 64,
        thumbs: [], //!IMPORTANT needs to be an array, will crash if undefined
    }
});
</script>

[tjenkinson]

Nice!

[vitordarela]

Hello everyone, I'm having problems following these steps, I'm getting the following error. "ClapprThumbnailsGeneratorPlugin requires HLS Playback"

But when I do a console.log(currentPlayback) I see my object .. and it contains _hls

[perohu]

I would like to add thumbnails for my hls live stream.

On the server i have files like

stream.m3u8
segment0.ts
thumb0.jpg
segment1.ts
thumb1.jpg
segment2.ts
thumb2.jpg

So for every segment I have a thumbnail that I want to show. Would that be possible?

Hello, can you please describe, how you create thumbs for all segment on the fly? I'm using ffmpeg to create live streams, but I don't know how to create those thumbs. Thank you

[rolandstarke]

Hello, I don't have a clever ffmpeg command.

When thumb0.jpg is requested the first time the server generates it with

ffmpeg -ss 00:00:00 -i  segment0.ts -vframes 1 -filter:v scale="-1:64" thumb0.jpg

It does not work that well on my raspberry pi, as the server needs to generate 100 thumbs on the fly the first time i visit the stream for a long time. Btw that could be an improvement for the library. Only load the thumbnails when needed. (many people probably don't hover over the timeline or when, only parts of it.)

[perohu]

Oh, I see, thank you.

[perohu]

Thanks a lot. For my livestream and VOD it is working now.

<div id="video"></div>


<script src="https://cdn.jsdelivr.net/npm/[email protected]/dist/clappr.min.js"></script>
<script src="https://cdn.jsdelivr.net/npm/[email protected]/dist/clappr-thumbnails-plugin.js"></script>
<script>
/* global Clappr */
var ClapprThumbnailsGeneratorPlugin = Clappr.UICorePlugin.extend({

    thumbnails: [],
    isBusy: false,
    thumbnailsPlugin: null,

    markUnbusy: function () {
        this.isBusy = false;
    },

    bindEvents: function () {
        this.thumbnailsPlugin = this.core.plugins.filter(plugin => plugin.name === 'scrub-thumbnails')[0];
        if (!this.thumbnailsPlugin) {
            console.error('ClapprThumbnailsGeneratorPlugin requires ClapprThumbnailsPlugin');
            return;
        }
        this.listenTo(this.core, Clappr.Events.CORE_READY, this.bindPlaybackEvents);
    },

    bindPlaybackEvents: function () {
        var currentPlayback = this.core.getCurrentPlayback();
        if (!currentPlayback._hls) {
            console.error('ClapprThumbnailsGeneratorPlugin requires HLS Playback');
            return;
        }

        currentPlayback._hls.on('hlsLevelUpdated', this.updateThumbnails.bind(this));

        if (currentPlayback.levels && currentPlayback.levels.length > 0) {
            this.updateThumbnails();
        }
    },

    updateThumbnails: function () {
        console.log('updateThumbnails called');
        var that = this;
        var currentPlayback = that.core.getCurrentPlayback();
        var level = (currentPlayback.levels[0] || {}).level;
        if (!level || !level.details || !level.details.fragments || !level.details.fragments.map) return;

        //get thumbnail paths and times from fragments
        var newThumnails = level.details.fragments.map(function (fragment) {
            return {
                time: fragment.start,
                url: fragment.baseurl + '/../' + fragment.relurl
                    .replace('segment', 'thumb')
                    .replace('.ts', '.jpg'), // segment0.ts --> thumb0.jpg
            };
        });

        //limit the thumbnails to a maximum of 200 images
        var useEachXThumbnails = Math.ceil(newThumnails.length / 200);
        newThumnails = newThumnails.filter(function (t, index) {
            return index % useEachXThumbnails === 0;
        });

        //check if there is a change. else we can stop here
        if (that.thumbnails.length === newThumnails.length
            && that.thumbnails.every(function (t, i) { return t.time === newThumnails[i].time && t.url === newThumnails[i].url; })
        ) {
            return;
        }

        //if the thumbnail plugin is still busy loading the images from the last update stop here
        if (that.isBusy) return;
        that.isBusy = true;

        console.log('updating thumbnails');
        that.thumbnailsPlugin.removeThumbnail(that.thumbnails).then(function () {
            that.thumbnails = newThumnails;
            return that.thumbnailsPlugin.addThumbnail(that.thumbnails);
        }).catch(console.error).then(that.markUnbusy.bind(that));
    }

});
</script>
<script>
var player = new Clappr.Player({
    source: "/camera/live/stream.m3u8",
    parentId: "#video",
    autoPlay: true,
    mute: true,
    persistConfig: false, /* do not save anything in localStorage */
    plugins: {
        core: [ClapprThumbnailsPlugin, ClapprThumbnailsGeneratorPlugin],
    },
    scrubThumbnails: {
        spotlightHeight: 64,
        thumbs: [], //!IMPORTANT needs to be an array, will crash if undefined
    }
});
</script>

Do you have a working live stream demo with this code? I'm trying to make this work, but I failed. "level.details" always "undefined" in the "updateThumbnails" function. I have to admit that I'm not an expert javascript programmer :)