From 7ef2de6b535aeece2d800f13247739657f99e8ec Mon Sep 17 00:00:00 2001 From: Mert <101130780+mertalev@users.noreply.github.com> Date: Wed, 17 Jun 2026 19:02:10 -0400 Subject: [PATCH] feat(server): add hls playlist hint (#29150) * add playlist hint * update api * unused import * update crf in tests --- mobile/openapi/lib/api/assets_api.dart | 14 +++- open-api/immich-openapi-specs.json | 9 +++ packages/sdk/src/fetch-client.ts | 8 ++- server/src/constants.ts | 6 ++ .../controllers/video-stream.controller.ts | 14 +++- server/src/dtos/streaming.dto.ts | 8 +++ server/src/enum.ts | 1 + server/src/services/hls.service.spec.ts | 66 +++++++++++++++++-- server/src/services/hls.service.ts | 39 +++++++++-- .../src/services/transcoding.service.spec.ts | 8 ++- server/src/services/transcoding.service.ts | 2 + 11 files changed, 153 insertions(+), 22 deletions(-) diff --git a/mobile/openapi/lib/api/assets_api.dart b/mobile/openapi/lib/api/assets_api.dart index e3bed71cd3..fa49d382b9 100644 --- a/mobile/openapi/lib/api/assets_api.dart +++ b/mobile/openapi/lib/api/assets_api.dart @@ -982,7 +982,9 @@ class AssetsApi { /// * [String] key: /// /// * [String] slug: - Future getMediaPlaylistWithHttpInfo(String id, String sessionId, int variantIndex, { String? key, String? slug, Future? abortTrigger, }) async { + /// + /// * [num] xImmichHlsPos: + Future getMediaPlaylistWithHttpInfo(String id, String sessionId, int variantIndex, { String? key, String? slug, num? xImmichHlsPos, Future? abortTrigger, }) async { // ignore: prefer_const_declarations final apiPath = r'/assets/{id}/video/stream/{sessionId}/{variantIndex}/playlist.m3u8' .replaceAll('{id}', id) @@ -1003,6 +1005,10 @@ class AssetsApi { queryParams.addAll(_queryParams('', 'slug', slug)); } + if (xImmichHlsPos != null) { + headerParams[r'x-immich-hls-pos'] = parameterToString(xImmichHlsPos); + } + const contentTypes = []; @@ -1033,8 +1039,10 @@ class AssetsApi { /// * [String] key: /// /// * [String] slug: - Future getMediaPlaylist(String id, String sessionId, int variantIndex, { String? key, String? slug, Future? abortTrigger, }) async { - final response = await getMediaPlaylistWithHttpInfo(id, sessionId, variantIndex, key: key, slug: slug, abortTrigger: abortTrigger,); + /// + /// * [num] xImmichHlsPos: + Future getMediaPlaylist(String id, String sessionId, int variantIndex, { String? key, String? slug, num? xImmichHlsPos, Future? abortTrigger, }) async { + final response = await getMediaPlaylistWithHttpInfo(id, sessionId, variantIndex, key: key, slug: slug, xImmichHlsPos: xImmichHlsPos, abortTrigger: abortTrigger,); if (response.statusCode >= HttpStatus.badRequest) { throw ApiException(response.statusCode, await _decodeBodyBytes(response)); } diff --git a/open-api/immich-openapi-specs.json b/open-api/immich-openapi-specs.json index 199dca8d76..8df2d80e69 100644 --- a/open-api/immich-openapi-specs.json +++ b/open-api/immich-openapi-specs.json @@ -4924,6 +4924,15 @@ "maximum": 9007199254740991, "type": "integer" } + }, + { + "name": "x-immich-hls-pos", + "required": false, + "in": "header", + "schema": { + "minimum": 0, + "type": "number" + } } ], "responses": { diff --git a/packages/sdk/src/fetch-client.ts b/packages/sdk/src/fetch-client.ts index b7202c2a1a..e230917b74 100644 --- a/packages/sdk/src/fetch-client.ts +++ b/packages/sdk/src/fetch-client.ts @@ -4452,12 +4452,13 @@ export function endSession({ id, key, sessionId, slug }: { /** * Get HLS media playlist */ -export function getMediaPlaylist({ id, key, sessionId, slug, variantIndex }: { +export function getMediaPlaylist({ id, key, sessionId, slug, variantIndex, xImmichHlsPos }: { id: string; key?: string; sessionId: string; slug?: string; variantIndex: number; + xImmichHlsPos?: number; }, opts?: Oazapfts.RequestOpts) { return oazapfts.ok(oazapfts.fetchBlob<{ status: 200; @@ -4466,7 +4467,10 @@ export function getMediaPlaylist({ id, key, sessionId, slug, variantIndex }: { key, slug }))}`, { - ...opts + ...opts, + headers: oazapfts.mergeHeaders(opts?.headers, { + "x-immich-hls-pos": xImmichHlsPos + }) })); } /** diff --git a/server/src/constants.ts b/server/src/constants.ts index 8a0f979ff0..7760fa8e8c 100644 --- a/server/src/constants.ts +++ b/server/src/constants.ts @@ -223,6 +223,12 @@ export const SUPPORTED_HWA_CODECS: Record = { + [VideoCodec.H264]: 23, + [VideoCodec.Hevc]: 28, + [VideoCodec.Vp9]: 31, + [VideoCodec.Av1]: 35, +}; export const HLS_INACTIVITY_TIMEOUT_MS = 5 * 60 * 1000; export const HLS_LEASE_DURATION_MS = 30 * 60 * 1000; export const HLS_PLAYLIST_CONTENT_TYPE = 'application/vnd.apple.mpegurl'; diff --git a/server/src/controllers/video-stream.controller.ts b/server/src/controllers/video-stream.controller.ts index c5f447af88..ed99f67296 100644 --- a/server/src/controllers/video-stream.controller.ts +++ b/server/src/controllers/video-stream.controller.ts @@ -6,6 +6,7 @@ import { HLS_PLAYLIST_CONTENT_TYPE } from 'src/constants'; import { Endpoint, HistoryBuilder } from 'src/decorators'; import { AuthDto } from 'src/dtos/auth.dto'; import { + HlsPlaylistHeaderDto, HlsSegmentHeaderDto, HlsSegmentParamDto, HlsSessionParamDto, @@ -50,8 +51,17 @@ export class VideoStreamController { description: 'Returns an HLS media playlist for one variant of the streaming session.', history: new HistoryBuilder().added('v3').alpha('v3'), }) - getMediaPlaylist(@Auth() auth: AuthDto, @Param() { id, sessionId }: HlsVariantParamDto) { - return this.service.getMediaPlaylist(auth, id, sessionId); + getMediaPlaylist( + @Auth() auth: AuthDto, + @Param() { id, sessionId, variantIndex }: HlsVariantParamDto, + @Headers() headers: HlsPlaylistHeaderDto, + ) { + try { + headers = HlsPlaylistHeaderDto.create(headers); + } catch (error) { + throw new ZodValidationException(error); + } + return this.service.getMediaPlaylist(auth, id, sessionId, variantIndex, headers[ImmichHeader.HlsPosition]); } @Get(':id/video/stream/:sessionId/:variantIndex/:filename') diff --git a/server/src/dtos/streaming.dto.ts b/server/src/dtos/streaming.dto.ts index e1d98745f8..c83c3e38af 100644 --- a/server/src/dtos/streaming.dto.ts +++ b/server/src/dtos/streaming.dto.ts @@ -32,3 +32,11 @@ const HlsSegmentHeaderSchema = z.object({ }); export class HlsSegmentHeaderDto extends createZodDto(HlsSegmentHeaderSchema) {} + +const HlsPlaylistHeaderSchema = z.object({ + // Lets the client hint at which segment will be loaded after the playlist. + // A position rather than a segment index since indices aren't comparable across variants. + [ImmichHeader.HlsPosition]: z.coerce.number().min(0).optional(), +}); + +export class HlsPlaylistHeaderDto extends createZodDto(HlsPlaylistHeaderSchema) {} diff --git a/server/src/enum.ts b/server/src/enum.ts index 45b77769c6..491beca10b 100644 --- a/server/src/enum.ts +++ b/server/src/enum.ts @@ -25,6 +25,7 @@ export enum ImmichHeader { Checksum = 'x-immich-checksum', CorrelationId = 'X-Correlation-ID', HlsInitSegment = 'x-immich-hls-msn', + HlsPosition = 'x-immich-hls-pos', } export enum ImmichQuery { diff --git a/server/src/services/hls.service.spec.ts b/server/src/services/hls.service.spec.ts index 9d9536f848..a18f7c6d22 100644 --- a/server/src/services/hls.service.spec.ts +++ b/server/src/services/hls.service.spec.ts @@ -8,6 +8,7 @@ import { newTestService, ServiceMocks } from 'test/utils'; // EXTINF values come from FFmpeg's playlist to enforce an exact match const eiffelExpectedMediaPlaylist = `#EXTM3U #EXT-X-VERSION:7 +#EXT-X-INDEPENDENT-SEGMENTS #EXT-X-TARGETDURATION:2 #EXT-X-MEDIA-SEQUENCE:0 #EXT-X-PLAYLIST-TYPE:VOD @@ -41,6 +42,7 @@ seg_11.m4s const waterfallExpectedMediaPlaylist = `#EXTM3U #EXT-X-VERSION:7 +#EXT-X-INDEPENDENT-SEGMENTS #EXT-X-TARGETDURATION:2 #EXT-X-MEDIA-SEQUENCE:0 #EXT-X-PLAYLIST-TYPE:VOD @@ -62,6 +64,7 @@ seg_5.m4s const trainExpectedMediaPlaylist = `#EXTM3U #EXT-X-VERSION:7 +#EXT-X-INDEPENDENT-SEGMENTS #EXT-X-TARGETDURATION:2 #EXT-X-MEDIA-SEQUENCE:0 #EXT-X-PLAYLIST-TYPE:VOD @@ -95,6 +98,7 @@ const sessionId = '00000000-0000-0000-0000-000000000000'; const eiffelExpectedMasterDisabled = `#EXTM3U #EXT-X-VERSION:7 +#EXT-X-INDEPENDENT-SEGMENTS #EXT-X-STREAM-INF:BANDWIDTH=1000000,RESOLUTION=480x852,CODECS="av01.0.04M.08,mp4a.40.2",VIDEO-RANGE=SDR,FRAME-RATE=24.910 ${sessionId}/0/playlist.m3u8 #EXT-X-STREAM-INF:BANDWIDTH=1200000,RESOLUTION=480x852,CODECS="hvc1.1.6.L90.B0,mp4a.40.2",VIDEO-RANGE=SDR,FRAME-RATE=24.910 @@ -117,6 +121,7 @@ ${sessionId}/8/playlist.m3u8 const eiffelExpectedMasterRkmpp = `#EXTM3U #EXT-X-VERSION:7 +#EXT-X-INDEPENDENT-SEGMENTS #EXT-X-STREAM-INF:BANDWIDTH=1200000,RESOLUTION=480x852,CODECS="hvc1.1.6.L90.B0,mp4a.40.2",VIDEO-RANGE=SDR,FRAME-RATE=24.910 ${sessionId}/1/playlist.m3u8 #EXT-X-STREAM-INF:BANDWIDTH=2500000,RESOLUTION=480x852,CODECS="avc1.64001e,mp4a.40.2",VIDEO-RANGE=SDR,FRAME-RATE=24.910 @@ -133,6 +138,7 @@ ${sessionId}/8/playlist.m3u8 const waterfallExpectedMasterDisabled = `#EXTM3U #EXT-X-VERSION:7 +#EXT-X-INDEPENDENT-SEGMENTS #EXT-X-STREAM-INF:BANDWIDTH=1000000,RESOLUTION=480x852,CODECS="av01.0.04M.08,mp4a.40.2",VIDEO-RANGE=SDR,FRAME-RATE=29.830 ${sessionId}/0/playlist.m3u8 #EXT-X-STREAM-INF:BANDWIDTH=1200000,RESOLUTION=480x852,CODECS="hvc1.1.6.L90.B0,mp4a.40.2",VIDEO-RANGE=SDR,FRAME-RATE=29.830 @@ -218,12 +224,58 @@ describe(HlsService.name, () => { it.each(fixtures)('matches FFmpeg for $data.originalPath', async ({ data, playlist }) => { mocks.access.asset.checkOwnerAccess.mockResolvedValue(new Set([assetId])); mocks.videoStream.getForMediaPlaylist.mockResolvedValue(data); - await expect(sut.getMediaPlaylist(auth, assetId, sessionId)).resolves.toBe(playlist); + await expect(sut.getMediaPlaylist(auth, assetId, sessionId, 0)).resolves.toBe(playlist); }); it('throws NotFoundException when the session/asset cannot be loaded', async () => { mocks.access.asset.checkOwnerAccess.mockResolvedValue(new Set([assetId])); - await expect(sut.getMediaPlaylist(auth, assetId, sessionId)).rejects.toBeInstanceOf(NotFoundException); + await expect(sut.getMediaPlaylist(auth, assetId, sessionId, 0)).rejects.toBeInstanceOf(NotFoundException); + }); + + it('prewarms transcoding at the segment containing the hinted position', async () => { + mocks.access.asset.checkOwnerAccess.mockResolvedValue(new Set([assetId])); + mocks.videoStream.getForMediaPlaylist.mockResolvedValue(eiffelTower); + await sut.getMediaPlaylist(auth, assetId, sessionId, 1, 10.5); + expect(mocks.websocket.serverSend).toHaveBeenCalledWith('HlsSegmentRequest', { + sessionId, + assetId, + variantIndex: 1, + segmentIndex: 5, + }); + }); + + it('prewarms from the last requested segment when no hint is given', async () => { + mocks.access.asset.checkOwnerAccess.mockResolvedValue(new Set([assetId])); + mocks.videoStream.getSession.mockResolvedValue({ id: sessionId, assetId } as never); + mocks.storage.checkFileExists.mockResolvedValue(true); + await sut.getSegment(auth, assetId, sessionId, 0, 'seg_5.m4s'); + + mocks.videoStream.getForMediaPlaylist.mockResolvedValue(eiffelTower); + await sut.getMediaPlaylist(auth, assetId, sessionId, 1); + expect(mocks.websocket.serverSend).toHaveBeenCalledWith('HlsSegmentRequest', { + sessionId, + assetId, + variantIndex: 1, + segmentIndex: 6, + }); + }); + + it('does not prewarm without a hint or prior segment traffic', async () => { + mocks.access.asset.checkOwnerAccess.mockResolvedValue(new Set([assetId])); + mocks.videoStream.getForMediaPlaylist.mockResolvedValue(eiffelTower); + await sut.getMediaPlaylist(auth, assetId, sessionId, 1); + expect(mocks.websocket.serverSend).not.toHaveBeenCalled(); + }); + + it('does not prewarm the variant the session is already playing', async () => { + mocks.access.asset.checkOwnerAccess.mockResolvedValue(new Set([assetId])); + mocks.videoStream.getSession.mockResolvedValue({ id: sessionId, assetId } as never); + mocks.storage.checkFileExists.mockResolvedValue(true); + await sut.getSegment(auth, assetId, sessionId, 1, 'seg_5.m4s'); + + mocks.videoStream.getForMediaPlaylist.mockResolvedValue(eiffelTower); + await sut.getMediaPlaylist(auth, assetId, sessionId, 1, 12.5); + expect(mocks.websocket.serverSend).not.toHaveBeenCalledWith('HlsSegmentRequest', expect.anything()); }); }); @@ -314,7 +366,7 @@ describe(HlsService.name, () => { ); }); - it('uses the target segment for init.mp4 when provided', async () => { + it('uses the initSegment hint for init.mp4', async () => { await sut.getSegment(auth, assetId, sessionId, variantIndex, 'init.mp4', 7); expect(mocks.websocket.serverSend).toHaveBeenCalledWith('HlsHeartbeat', { sessionId, @@ -323,18 +375,18 @@ describe(HlsService.name, () => { }); }); - it('prefers the target segment over the lastRequested + 1 fallback', async () => { + it('prefers the initSegment hint over the lastRequested + 1 fallback', async () => { await sut.getSegment(auth, assetId, sessionId, variantIndex, 'seg_5.m4s'); // fallback would be 6 mocks.websocket.serverSend.mockClear(); - await sut.getSegment(auth, assetId, sessionId, variantIndex, 'init.mp4', 12); + await sut.getSegment(auth, assetId, sessionId, variantIndex, 'init.mp4', 10); expect(mocks.websocket.serverSend).toHaveBeenCalledWith('HlsHeartbeat', { sessionId, variantIndex, - segmentIndex: 12, + segmentIndex: 10, }); }); - it('ignores the target segment for media segment requests (the filename wins)', async () => { + it('ignores the initSegment hint for media segment requests (the filename wins)', async () => { await sut.getSegment(auth, assetId, sessionId, variantIndex, 'seg_5.m4s', 99); expect(mocks.websocket.serverSend).toHaveBeenCalledWith('HlsHeartbeat', { sessionId, diff --git a/server/src/services/hls.service.ts b/server/src/services/hls.service.ts index 034f05bf6c..0cbb53e1cd 100644 --- a/server/src/services/hls.service.ts +++ b/server/src/services/hls.service.ts @@ -21,6 +21,7 @@ import { ImmichFileResponse } from 'src/utils/file'; import { getOutputSize } from 'src/utils/media'; type AssetWithStreamInfo = { videoStream: VideoStreamInfo & { timeBase: number }; packets: VideoPacketInfo }; +type Segmentation = { fps: number; framesPerSegment: number; segmentCount: number; segmentDuration: number }; type ApiSession = { lastRequestedSegment: number | null; lastVariantIndex: number | null }; @Injectable() @@ -71,7 +72,7 @@ export class HlsService extends BaseService { return this.generateMainPlaylist(sessionId, ffmpeg, asset); } - async getMediaPlaylist(auth: AuthDto, assetId: string, sessionId: string) { + async getMediaPlaylist(auth: AuthDto, assetId: string, sessionId: string, variantIndex: number, position?: number) { await this.requireAccess({ auth, permission: Permission.AssetView, ids: [assetId] }); const asset = await this.videoStreamRepository.getForMediaPlaylist(assetId, sessionId); @@ -79,7 +80,11 @@ export class HlsService extends BaseService { throw new NotFoundException('Asset not found or metadata not yet ready for streaming'); } - return this.generateMediaPlaylist(asset); + const segmentation = this.getSegmentation(asset); + const hintedSegment = position === undefined ? undefined : this.positionToSegmentIndex(segmentation, position); + this.prewarmVariant(assetId, sessionId, variantIndex, hintedSegment); + + return this.generateMediaPlaylist(asset, segmentation); } async getSegment( @@ -129,7 +134,7 @@ export class HlsService extends BaseService { const fps = ((asset.packets.packetCount * asset.videoStream.timeBase) / asset.packets.totalDuration).toFixed(3); const sourceResolution = Math.min(asset.videoStream.height, asset.videoStream.width); const targetResolution = Math.max(sourceResolution, HLS_VARIANTS[0].resolution); - const lines = ['#EXTM3U', `#EXT-X-VERSION:${HLS_VERSION}`]; + const lines = ['#EXTM3U', `#EXT-X-VERSION:${HLS_VERSION}`, '#EXT-X-INDEPENDENT-SEGMENTS']; for (let i = 0; i < HLS_VARIANTS.length; i++) { const { resolution, bitrate, codec, codecString } = HLS_VARIANTS[i]; if (resolution > targetResolution || !SUPPORTED_HWA_CODECS[ffmpeg.accel].includes(codec)) { @@ -143,24 +148,33 @@ export class HlsService extends BaseService { } lines.push(''); - if (lines.length === 3) { + if (lines.length === 4) { throw new NotFoundException('No supported variants for this video'); } return lines.join('\n'); } - private generateMediaPlaylist({ videoStream, packets }: AssetWithStreamInfo) { + private getSegmentation({ videoStream, packets }: AssetWithStreamInfo): Segmentation { const fps = (packets.packetCount * videoStream.timeBase) / packets.totalDuration; const framesPerSegment = Math.ceil(HLS_SEGMENT_DURATION * fps); - const fullSegmentDuration = framesPerSegment / fps; const segmentCount = Math.ceil(packets.outputFrames / framesPerSegment); + return { fps, framesPerSegment, segmentCount, segmentDuration: framesPerSegment / fps }; + } + + private positionToSegmentIndex({ segmentDuration, segmentCount }: Segmentation, position: number) { + return Math.min(Math.max(Math.floor(position / segmentDuration), 0), segmentCount - 1); + } + + private generateMediaPlaylist({ packets }: AssetWithStreamInfo, segmentation: Segmentation) { + const { fps, framesPerSegment, segmentCount, segmentDuration: fullSegmentDuration } = segmentation; const lastSegmentFrames = packets.outputFrames - framesPerSegment * (segmentCount - 1); const lastSegmentDuration = lastSegmentFrames / fps; const lines = [ '#EXTM3U', `#EXT-X-VERSION:${HLS_VERSION}`, + '#EXT-X-INDEPENDENT-SEGMENTS', `#EXT-X-TARGETDURATION:${HLS_SEGMENT_DURATION}`, '#EXT-X-MEDIA-SEQUENCE:0', '#EXT-X-PLAYLIST-TYPE:VOD', @@ -175,6 +189,19 @@ export class HlsService extends BaseService { return lines.join('\n'); } + private prewarmVariant(assetId: string, sessionId: string, variantIndex: number, hintedSegment?: number) { + const session = this.sessions.get(sessionId); + if (session?.lastVariantIndex === variantIndex) { + return; + } + + const nextSegment = session && session.lastRequestedSegment !== null ? session.lastRequestedSegment + 1 : undefined; + const segmentIndex = hintedSegment ?? nextSegment; + if (segmentIndex !== undefined) { + this.websocketRepository.serverSend('HlsSegmentRequest', { sessionId, assetId, variantIndex, segmentIndex }); + } + } + private getSegmentKey({ sessionId, variantIndex, segmentIndex }: ArgOf<'HlsSegmentResult'>) { return `${sessionId}:${variantIndex}:${segmentIndex}`; } diff --git a/server/src/services/transcoding.service.spec.ts b/server/src/services/transcoding.service.spec.ts index 349cba6a7d..8909a43342 100644 --- a/server/src/services/transcoding.service.spec.ts +++ b/server/src/services/transcoding.service.spec.ts @@ -381,8 +381,6 @@ describe(TranscodingService.name, () => { '50', '-keyint_min', '50', - '-crf', - '23', '-copyts', '-r', '50130000/2012441', @@ -417,6 +415,8 @@ describe(TranscodingService.name, () => { 'aac', '-preset', '12', + '-crf', + '35', '-svtav1-params', 'hierarchical-levels=3:lookahead=0:enable-tf=0:mbr=4000k', '-hls_segment_filename', @@ -436,6 +436,8 @@ describe(TranscodingService.name, () => { 'hvc1', '-preset', 'ultrafast', + '-crf', + '28', '-maxrate', '2500k', '-bufsize', @@ -459,6 +461,8 @@ describe(TranscodingService.name, () => { 'aac', '-preset', 'ultrafast', + '-crf', + '23', '-maxrate', '2500k', '-bufsize', diff --git a/server/src/services/transcoding.service.ts b/server/src/services/transcoding.service.ts index dae14f6257..c79eec7884 100644 --- a/server/src/services/transcoding.service.ts +++ b/server/src/services/transcoding.service.ts @@ -5,6 +5,7 @@ import { HLS_BACKPRESSURE_PAUSE_SEGMENTS, HLS_BACKPRESSURE_RESUME_SEGMENTS, HLS_CLEANUP_INTERVAL_MS, + HLS_CRF, HLS_INACTIVITY_TIMEOUT_MS, HLS_LEASE_DURATION_MS, HLS_SEGMENT_DURATION, @@ -221,6 +222,7 @@ export class TranscodingService extends BaseService { targetResolution: String(variant.resolution), maxBitrate: `${Math.round(variant.bitrate / 1000)}k`, gopSize: gop, + crf: HLS_CRF[variant.codec], }, this.videoInterfaces, { strictGop: true, lowLatency: true },