Image previews

The `sixel` flag enables sixel support in `ratatu-image`, which then
needs libsixel to build.

Additionally to enable the preview feature, one must add it to
config.json:
```json
    "image_preview": {
      "size": {
        "width": 60,
        "height": 10,
      }
    }
```

There is a `backend: "halfblocks"` option in `image_preview` to force a
specific backend rather than guessing it, which might not be 100%
reliable in all terminals.

Cargo.toml

@@ -67,10 +67,19 @@ features = ["e2e-encryption", "sled", "rustls-tls"]
 version = "1.24.1"
 features = ["macros", "net", "rt-multi-thread", "sync", "time"]
 
+[dependencies.ratatu-image]
+version = "0.1.1"
+optional = true
+features = ["serde", "rustix"]
+
 [dev-dependencies]
 lazy_static = "1.4.0"
 pretty_assertions = "1.4.0"
 
 [profile.release]
 lto = true
 incremental = false
+
+[features]
+default = ["sixel"]
+sixel = ["ratatu-image/sixel"]

flake.nix

@@ -29,11 +29,12 @@
         };
         devShell = mkShell {
           buildInputs = [
-            (rustNightly.override { extensions = [ "rust-src" ]; })
+            (rustNightly.override {
+              extensions = [ "rust-src" "rust-analyzer-preview" "rustfmt" "clippy" ]; 
+            })
             pkg-config
             cargo-tarpaulin
-            rust-analyzer
-            rustfmt
+            cargo-watch
           ];
         };
       });

src/base.rs

@@ -11,6 +11,7 @@ use std::sync::Arc;
 use std::time::{Duration, Instant};
 
 use emojis::Emoji;
+use ratatu_image::picker::Picker;
 use serde::{
     de::Error as SerdeError,
     de::Visitor,
@@ -490,6 +491,12 @@ pub enum IambError {
     /// A failure to access the system's clipboard.
     #[error("Could not use system clipboard data")]
     Clipboard,
+
+    #[error("IO error: {0}")]
+    IOError(#[from] std::io::Error),
+
+    #[error("Preview error: {0}")]
+    Preview(String),
 }
 
 impl From<IambError> for UIError<IambInfo> {
@@ -603,6 +610,10 @@ impl RoomInfo {
         self.messages.get(self.get_message_key(event_id)?)
     }
 
+    pub fn get_event_mut(&mut self, event_id: &EventId) -> Option<&mut Message> {
+        self.messages.get_mut(self.keys.get(event_id)?.to_message_key()?)
+    }
+
     /// Insert a reaction to a message.
     pub fn insert_reaction(&mut self, react: ReactionEvent) {
         match react {
@@ -825,6 +836,7 @@ pub struct ChatStore {
 
     /// Information gathered by the background thread.
     pub sync_info: SyncInfo,
+    pub picker: Option<Picker>,
 }
 
 impl ChatStore {
@@ -833,7 +845,7 @@ impl ChatStore {
         ChatStore {
             worker,
             settings,
-
+            picker: None,
             cmds: crate::commands::setup_commands(),
             emojis: emoji_map(),
 

src/config.rs

@@ -11,6 +11,7 @@ use std::process;
 
 use clap::Parser;
 use matrix_sdk::ruma::{OwnedRoomAliasId, OwnedRoomId, OwnedUserId, UserId};
+use ratatu_image::picker::BackendType;
 use serde::{de::Error as SerdeError, de::Visitor, Deserialize, Deserializer};
 use tracing::Level;
 use url::Url;
@@ -246,6 +247,26 @@ pub enum UserDisplayStyle {
     DisplayName,
 }
 
+#[derive(Clone, Deserialize)]
+pub struct ImagePreviewValues {
+    pub backend: Option<BackendType>,
+    pub size: ImagePreviewSize,
+}
+impl Default for ImagePreviewValues {
+    fn default() -> Self {
+        ImagePreviewValues {
+            backend: None,
+            size: ImagePreviewSize { width: 66, height: 10 },
+        }
+    }
+}
+
+#[derive(Clone, Deserialize)]
+pub struct ImagePreviewSize {
+    pub width: usize,
+    pub height: usize,
+}
+
 #[derive(Clone)]
 pub struct TunableValues {
     pub log_level: Level,
@@ -260,6 +281,7 @@ pub struct TunableValues {
     pub username_display: UserDisplayStyle,
     pub default_room: Option<String>,
     pub open_command: Option<Vec<String>>,
+    pub image_preview: Option<ImagePreviewValues>,
 }
 
 #[derive(Clone, Default, Deserialize)]
@@ -276,6 +298,7 @@ pub struct Tunables {
     pub username_display: Option<UserDisplayStyle>,
     pub default_room: Option<String>,
     pub open_command: Option<Vec<String>>,
+    pub image_preview: Option<ImagePreviewValues>,
 }
 
 impl Tunables {
@@ -295,6 +318,7 @@ impl Tunables {
             username_display: self.username_display.or(other.username_display),
             default_room: self.default_room.or(other.default_room),
             open_command: self.open_command.or(other.open_command),
+            image_preview: self.image_preview.or(other.image_preview),
         }
     }
 
@@ -312,6 +336,7 @@ impl Tunables {
             username_display: self.username_display.unwrap_or_default(),
             default_room: self.default_room,
             open_command: self.open_command,
+            image_preview: self.image_preview,
         }
     }
 }

src/main.rs

@@ -27,6 +27,7 @@ use std::sync::Arc;
 use std::time::Duration;
 
 use clap::Parser;
+use ratatu_image::picker::Picker;
 use tokio::sync::Mutex as AsyncMutex;
 use tracing_subscriber::FmtSubscriber;
 
@@ -68,6 +69,7 @@ mod commands;
 mod config;
 mod keybindings;
 mod message;
+mod preview;
 mod util;
 mod windows;
 mod worker;
@@ -264,9 +266,16 @@ impl Application {
         let bindings = KeyManager::new(bindings);
 
         let mut locked = store.lock().await;
+
+        if settings.tunables.image_preview.is_some() {
+            let picker = Picker::from_termios(None).unwrap();
+            locked.application.picker = Some(picker);
+        }
+
         let screen = setup_screen(settings, locked.deref_mut())?;
 
         let worker = locked.application.worker.clone();
+
         drop(locked);
 
         let actstack = VecDeque::new();

src/message/mod.rs

@@ -51,7 +51,9 @@ use modalkit::tui::{
 };
 
 use modalkit::editing::{base::ViewportContext, cursor::Cursor};
+use ratatu_image::backend::FixedBackend;
 
+use crate::config::ImagePreviewSize;
 use crate::{
     base::{IambResult, RoomInfo},
     config::ApplicationSettings,
@@ -584,20 +586,36 @@ impl<'a> MessageFormatter<'a> {
     }
 }
 
+pub enum ImageBackend {
+    None,
+    Downloading(ImagePreviewSize),
+    Preparing(ImagePreviewSize),
+    Loaded(Box<dyn FixedBackend>),
+    Error(String),
+}
+
 pub struct Message {
     pub event: MessageEvent,
     pub sender: OwnedUserId,
     pub timestamp: MessageTimeStamp,
     pub downloaded: bool,
     pub html: Option<StyleTree>,
+    pub image_backend: ImageBackend,
 }
 
 impl Message {
     pub fn new(event: MessageEvent, sender: OwnedUserId, timestamp: MessageTimeStamp) -> Self {
         let html = event.html();
         let downloaded = false;
 
-        Message { event, sender, timestamp, downloaded, html }
+        Message {
+            event,
+            sender,
+            timestamp,
+            downloaded,
+            html,
+            image_backend: ImageBackend::None,
+        }
     }
 
     pub fn reply_to(&self) -> Option<OwnedEventId> {
@@ -683,6 +701,29 @@ impl Message {
         }
     }
 
+    pub fn line_preview(
+        &self,
+        prev: Option<&Message>,
+        vwctx: &ViewportContext<MessageCursor>,
+    ) -> Option<(&Box<dyn FixedBackend>, u16, u16)> {
+        if let ImageBackend::Loaded(backend) = &self.image_backend {
+            let width = vwctx.get_width();
+            // The x position where get_render_format would render the text.
+            let x = (if USER_GUTTER + MIN_MSG_LEN <= width {
+                USER_GUTTER
+            } else {
+                0
+            } + 1) as u16;
+            // See get_render_format; account for possible "date" line.
+            let date_y = match &prev {
+                Some(prev) if !prev.timestamp.same_day(&self.timestamp) => 1,
+                _ => 0,
+            };
+            return Some((backend, x, date_y));
+        }
+        None
+    }
+
     pub fn show<'a>(
         &'a self,
         prev: Option<&Message>,
@@ -793,6 +834,22 @@ impl Message {
                 msg.to_mut().push_str(" \u{2705}");
             }
 
+            if let Some(placeholder) = match &self.image_backend {
+                ImageBackend::None => None,
+                ImageBackend::Downloading(image_preview_size) => {
+                    Some(Message::placeholder_frame(Some("Downloading..."), image_preview_size))
+                },
+                ImageBackend::Preparing(image_preview_size) => {
+                    Some(Message::placeholder_frame(Some("Preparing..."), image_preview_size))
+                },
+                ImageBackend::Loaded(backend) => {
+                    Some(Message::placeholder_frame(None, &backend.rect().into()))
+                },
+                ImageBackend::Error(err) => Some(format!("[Image error: {err}]")),
+            } {
+                msg.to_mut().insert_str(0, &placeholder);
+            }
+
             wrapped_text(msg, width, style)
         }
     }
@@ -805,6 +862,23 @@ impl Message {
         settings.get_user_span(self.sender.as_ref(), info)
     }
 
+    fn placeholder_frame(text: Option<&str>, image_preview_size: &ImagePreviewSize) -> String {
+        let ImagePreviewSize { width, height } = image_preview_size;
+        let mut placeholder = "\u{230c}".to_string();
+        placeholder.push_str(&" ".repeat(width - 2));
+        placeholder.push_str("\u{230d}\n");
+        placeholder.push_str(" ");
+        if let Some(text) = text {
+            placeholder.push_str(text);
+        }
+
+        placeholder.push_str(&"\n".repeat(height - 2));
+        placeholder.push_str("\u{230e}");
+        placeholder.push_str(&" ".repeat(width - 2));
+        placeholder.push_str("\u{230f}\n");
+        placeholder
+    }
+
     fn show_sender<'a>(
         &'a self,
         prev: Option<&Message>,