Skip to content

Commit

Permalink
Add H.264 decoder
Browse files Browse the repository at this point in the history 
This adds an H.264 decoder based on WebCodecs.
  • Loading branch information
@any1
any1 committed Jul 8, 2024
1 parent 1c93b9f commit b7e21db
Show file tree
Hide file tree
Showing 4 changed files with 395 additions and 0 deletions.
346 changes: 346 additions & 0 deletions core/decoders/h264.js
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,346 @@
/*
* noVNC: HTML5 VNC client
* Copyright (C) 2024 The noVNC Authors
* Licensed under MPL 2.0 (see LICENSE.txt)
*
* See README.md for usage and integration instructions.
*
*/

import * as Log from '../util/logging.js';

class H264Parser {
constructor(data) {
this._data = data;
this._index = 0;
this.profileIdc = null;
this.constraintSet = null;
this.levelIdc = null;
}

_getStartSequenceLen(index) {
let data = this._data;
if (data[index + 0] == 0 && data[index + 1] == 0 && data[index + 2] == 0 && data[index + 3] == 1) {
return 4;
}
if (data[index + 0] == 0 && data[index + 1] == 0 && data[index + 2] == 1) {
return 3;
}
return 0;
}

_indexOfNextNalUnit(index) {
let data = this._data;
for (let i = index; i < data.length; ++i) {
if (this._getStartSequenceLen(i) != 0) {
return i;
}
}
return -1;
}

_parseSps(index) {
this.profileIdc = this._data[index];
this.constraintSet = this._data[index + 1];
this.levelIdc = this._data[index + 2];
}

_parseNalUnit(index) {
const firstByte = this._data[index];
if (firstByte & 0x80) {
throw new Error('H264 parsing sanity check failed, forbidden zero bit is set');
}
const unitType = firstByte & 0x1f;

switch (unitType) {
case 1: // coded slice, non-idr
return { slice: true };
case 5: // coded slice, idr
return { slice: true, key: true };
case 6: // sei
return {};
case 7: // sps
this._parseSps(index + 1);
return {};
case 8: // pps
return {};
default:
Log.Warn("Unhandled unit type: ", unitType);
break;
}
return {};
}

parse() {
const startIndex = this._index;
let isKey = false;

while (this._index < this._data.length) {
const startSequenceLen = this._getStartSequenceLen(this._index);
if (startSequenceLen == 0) {
throw new Error('Invalid start sequence in bit stream');
}

const { slice, key } = this._parseNalUnit(this._index + startSequenceLen);

let nextIndex = this._indexOfNextNalUnit(this._index + startSequenceLen);
if (nextIndex == -1) {
this._index = this._data.length;
} else {
this._index = nextIndex;
}

if (key) {
isKey = true;
}
if (slice) {
break;
}
}

if (startIndex === this._index) {
return null;
}

return {
frame: this._data.subarray(startIndex, this._index),
key: isKey,
};
}
}

class H264Context {
constructor(owner, width, height) {
this.lastUsed = 0;
this._owner = owner;
this._width = width;
this._height = height;
this._profileIdc = null;
this._constraintSet = null;
this._levelIdc = null;
this._decoder = null;
this._pendingFrames = [];
}

_handleFrame(frame) {
let pending = this._pendingFrames.shift();
if (pending === undefined) {
throw new Error("Pending frame queue empty when receiving frame from decoder");
}

if (pending.timestamp != frame.timestamp) {
throw new Error("Video frame timestamp mismatch. Expected " +
frame.timestamp + " but but got " + pending.timestamp);
}

pending.frame = frame;
pending.ready = true;
pending.resolve();

if (!pending.keep) {
frame.close();
}
}

_flushPendingFrames() {
let pendingFrames = this._pendingFrames;
this._pendingFrames = [];

for (const pending of pendingFrames) {
pending.failed = true;
pending.resolve();
}
}

_handleError(e) {
Log.Warn("Failed to decode frame:", e);
this._flushPendingFrames();
this._owner._requestKeyFrame();
}

_configureDecoder(profileIdc, constraintSet, levelIdc) {
if (this._decoder === null || this._decoder.state === 'closed') {
this._decoder = new VideoDecoder({
output: frame => this._handleFrame(frame),
error: e => this._handleError(e),
});
}
const codec = 'avc1.' +
profileIdc.toString(16).padStart(2, '0') +
constraintSet.toString(16).padStart(2, '0') +
levelIdc.toString(16).padStart(2, '0');
this._decoder.configure({
codec: codec,
codecWidth: this._width,
codecHeight: this._height,
optimizeForLatency: true,
});
}

_preparePendingFrame(timestamp) {
let pending = {
timestamp: timestamp,
promise: null,
resolve: null,
frame: null,
ready: false,
failed: false,
keep: false,
};
pending.promise = new Promise((resolve) => {
pending.resolve = resolve;
});
this._pendingFrames.push(pending);

return pending;
}

decode(payload) {
let parser = new H264Parser(payload);
let result = null;

// Ideally, this timestamp should come from the server, but we'll just
// approximate it instead.
let timestamp = Math.round(window.performance.now() * 1e3);

while (true) {
let encodedFrame = parser.parse();
if (encodedFrame === null) {
break;
}

if (parser.profileIdc !== null) {
self._profileIdc = parser.profileIdc;
self._constraintSet = parser.constraintSet;
self._levelIdc = parser.levelIdc;
}

if (this._decoder === null || this._decoder.state !== 'configured') {
if (!encodedFrame.key) {
Log.Warn("Missing key frame. Can't decode until one arrives");
continue;
}
if (self._profileIdc === null) {
Log.Warn('Cannot config decoder. Have not received SPS and PPS yet.');
continue;
}
this._configureDecoder(self._profileIdc, self._constraintSet,
self._levelIdc);
}

result = this._preparePendingFrame(timestamp);

const chunk = new EncodedVideoChunk({
timestamp: timestamp,
type: encodedFrame.key ? 'key' : 'delta',
data: encodedFrame.frame,
});

try {
this._decoder.decode(chunk);
} catch (e) {
Log.Warn("Failed to decode:", e);
}

// If the decoder wants to use the timestamp for approximating a
// minimum frame duration, it's better to have a non-zero difference
// between frames, so we will assume 60 Hz refresh rate as it is the
// most common one.
timestamp += 16667;
}

// We only keep last frame of each payload
if (result !== null) {
result.keep = true;
}

return result;
}
}

export default class H264Decoder {
constructor(requestKeyFrame) {
this._tick = 0;
this._contexts = {};
this._requestKeyFrame = requestKeyFrame;
}

static check() {
return 'VideoDecoder' in window;
}

_contextId(x, y, width, height) {
return [x, y, width, height].join(',');
}

_findOldestContextId() {
let oldestTick = 0;
let oldestKey = undefined;
for (const [key, value] of Object.entries(this._contexts)) {
if (value.lastUsed < oldestTick) {
oldestTick = value.lastUsed;
oldestKey = key;
}
}
return oldestKey;
}

_createContext(x, y, width, height) {
const maxContexts = 64;
if (Object.keys(this._contexts).length >= maxContexts) {
let oldestContextId = this._findOldestContextId();
delete this._contexts[oldestContextId];
}
let context = new H264Context(this, width, height);
this._contexts[this._contextId(x, y, width, height)] = context;
return context;
}

_getContext(x, y, width, height) {
let context = this._contexts[this._contextId(x, y, width, height)];
return context !== undefined ? context : this._createContext(x, y, width, height);
}

_resetContext(x, y, width, height) {
delete this._contexts[this._contextId(x, y, width, height)];
}

_resetAllContexts() {
this._contexts = {};
}

decodeRect(x, y, width, height, sock, display, depth) {
const resetContextFlag = 1;
const resetAllContextsFlag = 2;

if (sock.rQwait("h264 header", 8)) {
return false;
}

const length = sock.rQshift32();
const flags = sock.rQshift32();

if (sock.rQwait("h264 payload", length, 8)) {
return false;
}

if (flags & resetAllContextsFlag) {
this._resetAllContexts();
} else if (flags & resetContextFlag) {
this._resetContext(this._contextId(x, y, width, height));
}

let context = this._getContext(x, y, width, height);
context.lastUsed = this._tick++;

if (length !== 0) {
let payload = sock.rQshiftBytes(length, false);
let frame = context.decode(payload);
if (frame !== null) {
display.videoFrame(x, y, width, height, frame);
}
}

return true;
}
}
Loading

0 comments on commit b7e21db

Please sign in to comment.