fix: show the picture in the playing track's file on the now-playing screen

Nextcloud Music gives every song of an album the album's cover id, so
tracks with their own art all showed the album cover. The player now
reads the picture embedded in each file it decodes and reports it once
per track, or reports that there is none. The now-playing art, backdrop
and palette use it, and wait for that answer so they do not show the
album cover first and then change. Lists and the queue keep the
server's cover.

Pictures are read where each file's decoder hands on its stream:
further down, a file without a picture still carries the previous
file's tags after a gapless switch.
This commit is contained in:
Ansho Rei
2026-11-04 20:22:03 +01:00
parent b5ef9fbb13
commit 6997e7cd93
6 changed files with 278 additions and 40 deletions
+176 -7
View File
@@ -2,6 +2,7 @@
//! track can be handed to playbin in `about-to-finish`, which keeps playback
//! gapless.
use std::collections::HashMap;
use std::sync::atomic::{AtomicBool, Ordering};
use std::sync::{Arc, Mutex, MutexGuard};
use std::thread::JoinHandle;
@@ -11,13 +12,17 @@ use anyhow::{Context, Result};
use gstreamer as gst;
use gstreamer::prelude::*;
#[derive(Debug, Clone, Copy)]
#[derive(Debug, Clone)]
pub enum PlayerEvent {
Position(f64),
Duration(f64),
Paused(bool),
/// Index into the queue, None when playback has stopped.
Track(Option<usize>),
/// The picture embedded in a track's file, or None once it is clear the
/// file has none. Sent once per track started, after its `Track`. A
/// picture that turns up late is still sent, after the None.
Art(usize, Option<Vec<u8>>),
}
#[derive(Default)]
@@ -33,14 +38,21 @@ struct Queue {
seek_to: Option<f64>,
}
/// Pictures found in the files being decoded, by URI.
type Pictures = Arc<Mutex<HashMap<String, Vec<u8>>>>;
pub struct Player {
playbin: gst::Element,
queue: Arc<Mutex<Queue>>,
pictures: Pictures,
stop: Arc<AtomicBool>,
bus_thread: Option<JoinHandle<()>>,
}
const TICK: Duration = Duration::from_millis(250);
/// How long after a track starts its tags can still bring a picture. Tags
/// come ahead of the audio, so this is mostly a slow server's margin.
const ART_WAIT: Duration = Duration::from_millis(600);
impl Player {
pub fn spawn(on_event: impl Fn(PlayerEvent) + Send + 'static) -> Result<Player> {
@@ -71,15 +83,19 @@ impl Player {
None
});
let pictures = Pictures::default();
watch_pictures(&playbin, &pictures);
let stop = Arc::new(AtomicBool::new(false));
let bus_thread = std::thread::Builder::new().name("gst-bus".into()).spawn({
let (playbin, queue, stop) = (playbin.clone(), queue.clone(), stop.clone());
move || watch_bus(&playbin, &queue, &stop, &on_event)
let (playbin, queue, pictures, stop) = (playbin.clone(), queue.clone(), pictures.clone(), stop.clone());
move || watch_bus(&playbin, &queue, &pictures, &stop, &on_event)
})?;
Ok(Player {
playbin,
queue,
pictures,
stop,
bus_thread: Some(bus_thread),
})
@@ -147,6 +163,7 @@ impl Player {
/// to Null flushes the bus, so nothing from the old track reports later.
pub fn stop(&self) {
*lock(&self.queue) = Queue::default();
lock(&self.pictures).clear();
let _ = self.playbin.set_state(gst::State::Null);
}
@@ -213,8 +230,53 @@ impl Drop for Player {
}
}
fn lock(q: &Mutex<Queue>) -> MutexGuard<'_, Queue> {
q.lock().unwrap_or_else(|e| e.into_inner())
fn lock<T>(m: &Mutex<T>) -> MutexGuard<'_, T> {
m.lock().unwrap_or_else(|e| e.into_inner())
}
/// Records the picture in each file playbin decodes, keyed by its URI. Read
/// where each file's decoder hands on its stream, because further down the
/// tags of a file without a picture still carry the previous file's, once
/// playback moves on gaplessly.
fn watch_pictures(playbin: &gst::Element, pictures: &Pictures) {
let Some(bin) = playbin.downcast_ref::<gst::Bin>() else {
return;
};
let pictures = pictures.clone();
bin.connect_deep_element_added(move |_, _, element| {
let decoder = element
.factory()
.is_some_and(|f| matches!(f.name().as_str(), "uridecodebin" | "uridecodebin3"));
// playbin adds the same decoder again for later tracks.
// SAFETY: the key is only ever set to a bool, here.
if !decoder || unsafe { element.data::<bool>("remus-pictures") }.is_some() {
return;
}
unsafe { element.set_data("remus-pictures", true) };
let pictures = pictures.clone();
element.connect_pad_added(move |decoder, pad| {
let decoder = decoder.downgrade();
let pictures = pictures.clone();
pad.add_probe(gst::PadProbeType::EVENT_DOWNSTREAM, move |_, info| {
if let Some(gst::PadProbeData::Event(ev)) = &info.data
&& let gst::EventView::Tag(t) = ev.view()
&& let Some(bytes) = embedded_picture(t.tag())
&& let Some(uri) = decoder.upgrade().and_then(|d| d.property::<Option<String>>("uri"))
{
lock(&pictures).entry(uri).or_insert(bytes);
}
gst::PadProbeReturn::Ok
});
});
});
}
/// The track whose picture is still being looked for.
struct ArtWait {
index: usize,
since: Instant,
/// Already reported as without a picture; one turning up later is still sent.
told_none: bool,
}
fn start(playbin: &gst::Element, url: &str) {
@@ -246,7 +308,13 @@ fn seek_to(playbin: &gst::Element, seconds: f64) {
}
}
fn watch_bus(playbin: &gst::Element, queue: &Mutex<Queue>, stop: &AtomicBool, emit: &dyn Fn(PlayerEvent)) {
fn watch_bus(
playbin: &gst::Element,
queue: &Mutex<Queue>,
pictures: &Pictures,
stop: &AtomicBool,
emit: &dyn Fn(PlayerEvent),
) {
let Some(bus) = playbin.bus() else { return };
let mut last_tick = Instant::now();
let mut duration = 0.0;
@@ -259,8 +327,12 @@ fn watch_bus(playbin: &gst::Element, queue: &Mutex<Queue>, stop: &AtomicBool, em
last_track = Some(t);
emit(PlayerEvent::Track(t));
emit(PlayerEvent::Position(0.0));
true
} else {
false
}
};
let mut art_wait: Option<ArtWait> = None;
while !stop.load(Ordering::Relaxed) {
if let Some(msg) = bus.timed_pop(gst::ClockTime::from_mseconds(TICK.as_millis() as u64)) {
use gst::MessageView;
@@ -278,7 +350,24 @@ fn watch_bus(playbin: &gst::Element, queue: &Mutex<Queue>, stop: &AtomicBool, em
(q.current, started)
};
duration = 0.0;
report_track(current, started);
if report_track(current, started) {
art_wait = current.map(|index| ArtWait {
index,
since: Instant::now(),
told_none: false,
});
// Keep only what this track and the next may still need.
let keep: Vec<String> = {
let q = lock(queue);
current
.into_iter()
.flat_map(|i| q.urls.get(i..=i + 1))
.flatten()
.cloned()
.collect()
};
lock(pictures).retain(|uri, _| keep.contains(uri));
}
}
MessageView::StateChanged(s) if s.src() == Some(playbin.upcast_ref()) => match s.current() {
gst::State::Playing => emit(PlayerEvent::Paused(false)),
@@ -303,6 +392,7 @@ fn watch_bus(playbin: &gst::Element, queue: &Mutex<Queue>, stop: &AtomicBool, em
}
let _ = playbin.set_state(gst::State::Ready);
report_track(None, false);
art_wait = None;
}
// A failing source posts several errors. Once the first has moved
// on to the next track, the source is out of playbin and the rest
@@ -336,11 +426,23 @@ fn watch_bus(playbin: &gst::Element, queue: &Mutex<Queue>, stop: &AtomicBool, em
} else {
let _ = playbin.set_state(gst::State::Ready);
report_track(None, false);
art_wait = None;
}
}
_ => {}
}
}
if let Some(w) = &mut art_wait {
let url = lock(queue).urls.get(w.index).cloned();
let picture = url.and_then(|u| lock(pictures).get(&u).cloned());
if picture.is_some() {
emit(PlayerEvent::Art(w.index, picture));
art_wait = None;
} else if !w.told_none && w.since.elapsed() >= ART_WAIT {
w.told_none = true;
emit(PlayerEvent::Art(w.index, None));
}
}
if last_tick.elapsed() >= TICK {
last_tick = Instant::now();
if playbin.current_state() == gst::State::Playing {
@@ -358,6 +460,22 @@ fn watch_bus(playbin: &gst::Element, queue: &Mutex<Queue>, stop: &AtomicBool, em
}
}
/// The front cover among the pictures in `tags`, else the first picture.
fn embedded_picture(tags: &gst::TagListRef) -> Option<Vec<u8>> {
// GST_TAG_IMAGE_TYPE_FRONT_COVER in gst-tag's GstTagImageType.
const FRONT_COVER: i32 = 1;
let is_front = |s: &gst::Sample| {
s.info()
.and_then(|i| i.value("image-type").ok())
.and_then(|v| gst::glib::EnumValue::from_value(v))
.is_some_and(|(_, v)| v.value() == FRONT_COVER)
};
let pictures: Vec<gst::Sample> = tags.iter_tag::<gst::tags::Image>().map(|v| v.get()).collect();
let sample = pictures.iter().find(|s| is_front(s)).or(pictures.first())?;
let map = sample.buffer()?.map_readable().ok()?;
Some(map.as_slice().to_vec())
}
/// Removes the query string from any URL in `text`. Stream URLs carry the
/// Subsonic auth token, and GStreamer's HTTP errors quote the URL.
fn strip_queries(text: &str) -> String {
@@ -425,6 +543,57 @@ mod tests {
format!("file://{}", path.display())
}
/// Puts an ID3v2.3 tag holding `picture` as the front cover before the
/// file at `uri`, as a tagged MP3 would carry it.
fn with_id3_picture(uri: &str, picture: &[u8]) {
let path = uri.strip_prefix("file://").unwrap();
let mut body = vec![0u8]; // Latin-1 text
body.extend_from_slice(b"image/png\0");
body.push(3); // front cover
body.push(0); // empty description
body.extend_from_slice(picture);
let mut frame = b"APIC".to_vec();
frame.extend_from_slice(&(body.len() as u32).to_be_bytes());
frame.extend_from_slice(&[0, 0]);
frame.extend_from_slice(&body);
let n = frame.len() as u32;
let syncsafe = [(n >> 21) & 0x7f, (n >> 14) & 0x7f, (n >> 7) & 0x7f, n & 0x7f].map(|b| b as u8);
let mut tagged = b"ID3\x03\x00\x00".to_vec();
tagged.extend_from_slice(&syncsafe);
tagged.extend_from_slice(&frame);
tagged.extend_from_slice(&std::fs::read(path).unwrap());
std::fs::write(path, tagged).unwrap();
}
#[test]
fn reports_the_picture_in_each_file() {
// SAFETY: set before any thread reads the environment in this test.
unsafe { std::env::set_var("REMUS_AUDIO_SINK", "fakesink") };
let dir = std::env::temp_dir().join(format!("remus-player-art-{}", std::process::id()));
std::fs::create_dir_all(&dir).unwrap();
let urls = vec![silent_wav(&dir, "a.wav", 1.0), silent_wav(&dir, "b.wav", 1.0)];
let mut png = Vec::new();
image::RgbImage::from_pixel(4, 4, image::Rgb([200, 30, 30]))
.write_to(&mut std::io::Cursor::new(&mut png), image::ImageFormat::Png)
.unwrap();
with_id3_picture(&urls[0], &png);
let (tx, rx) = mpsc::channel();
let player = Player::spawn(move |ev| {
if let PlayerEvent::Art(i, bytes) = ev {
let _ = tx.send((i, bytes));
}
})
.unwrap();
player.play_list(&urls, 0);
let first = rx.recv_timeout(Duration::from_secs(5));
let second = rx.recv_timeout(Duration::from_secs(5));
drop(player);
let _ = std::fs::remove_dir_all(&dir);
assert_eq!(first, Ok((0, Some(png))), "the tagged file's picture");
assert_eq!(second, Ok((1, None)), "the plain file has none");
}
#[test]
fn plays_a_queue_through_to_the_end() {
// SAFETY: set before any thread reads the environment in this test.