Blurhash support (#701)

* Generate blurhash client side

* Make blurhash generation faster

* Simple blurhash display support

* Make image display simpler

* Support non square images

* Don't attach video blurhash to thumbnail

* Add video display support

* Ignore alt tag missing warning

Co-authored-by: Ajay Bura <32841439+ajbura@users.noreply.github.com>

src/app/molecules/media/Media.jsx

@@ -4,6 +4,7 @@ import './Media.scss';
 
 import encrypt from 'browser-encrypt-attachment';
 
+import { BlurhashCanvas } from 'react-blurhash';
 import Text from '../../atoms/text/Text';
 import IconButton from '../../atoms/button/IconButton';
 import Spinner from '../../atoms/spinner/Spinner';
@@ -154,7 +155,7 @@ File.propTypes = {
 };
 
 function Image({
-  name, width, height, link, file, type,
+  name, width, height, link, file, type, blurhash,
 }) {
   const [url, setUrl] = useState(null);
 
@@ -175,6 +176,7 @@ function Image({
     <div className="file-container">
       <FileHeader name={name} link={url || link} type={type} external />
       <div style={{ height: width !== null ? getNativeHeight(width, height) : 'unset' }} className="image-container">
+        { blurhash && <BlurhashCanvas hash={blurhash} punch={1} />}
         { url !== null && <img src={url || link} alt={name} />}
       </div>
     </div>
@@ -185,6 +187,7 @@ Image.defaultProps = {
   width: null,
   height: null,
   type: '',
+  blurhash: '',
 };
 Image.propTypes = {
   name: PropTypes.string.isRequired,
@@ -193,6 +196,7 @@ Image.propTypes = {
   link: PropTypes.string.isRequired,
   file: PropTypes.shape({}),
   type: PropTypes.string,
+  blurhash: PropTypes.string,
 };
 
 function Sticker({
@@ -278,8 +282,8 @@ Audio.propTypes = {
 };
 
 function Video({
-  name, link, thumbnail,
-  width, height, file, type, thumbnailFile, thumbnailType,
+  name, link, thumbnail, thumbnailFile, thumbnailType,
+  width, height, file, type, blurhash,
 }) {
   const [isLoading, setIsLoading] = useState(false);
   const [url, setUrl] = useState(null);
@@ -315,10 +319,14 @@ function Video({
       <div
         style={{
           height: width !== null ? getNativeHeight(width, height) : 'unset',
-          backgroundImage: thumbUrl === null ? 'none' : `url(${thumbUrl}`,
         }}
         className="video-container"
       >
+        { url === null && blurhash && <BlurhashCanvas hash={blurhash} punch={1} />}
+        { url === null && thumbUrl !== null && (
+          /* eslint-disable-next-line jsx-a11y/alt-text */
+          <img src={thumbUrl} />
+        )}
         { url === null && isLoading && <Spinner size="small" /> }
         { url === null && !isLoading && <IconButton onClick={handlePlayVideo} tooltip="Play video" src={PlaySVG} />}
         { url !== null && (
@@ -336,20 +344,22 @@ Video.defaultProps = {
   height: null,
   file: null,
   thumbnail: null,
-  type: '',
   thumbnailType: null,
   thumbnailFile: null,
+  type: '',
+  blurhash: null,
 };
 Video.propTypes = {
   name: PropTypes.string.isRequired,
   link: PropTypes.string.isRequired,
   thumbnail: PropTypes.string,
+  thumbnailFile: PropTypes.shape({}),
+  thumbnailType: PropTypes.string,
   width: PropTypes.number,
   height: PropTypes.number,
   file: PropTypes.shape({}),
   type: PropTypes.string,
-  thumbnailFile: PropTypes.shape({}),
-  thumbnailType: PropTypes.string,
+  blurhash: PropTypes.string,
 };
 
 export {

src/app/molecules/media/Media.scss

@@ -33,6 +33,8 @@
   font-size: 0;
   line-height: 0;
 
+  position: relative;
+
   display: flex;
   justify-content: center;
   align-items: center;
@@ -42,6 +44,19 @@
   background-size: cover;
 }
 
+.image-container,
+.video-container {
+  & img,
+  & canvas {
+    position: absolute;
+    max-width: unset !important;
+    width: 100% !important;
+    height: 100%;
+    border-radius: 0 !important;
+    margin: 0 !important;
+  }
+}
+
 .sticker-container {
   display: inline-flex;
   max-width: 128px;
@@ -51,25 +66,17 @@
   }
 }
 
-.image-container {
-  & img {
-    max-width: unset !important;
-    width: 100% !important;
-    border-radius: 0 !important;
-    margin: 0 !important;
-  }
-}
-
 .video-container {
   & .ic-btn-surface {
     background-color: var(--bg-surface-low);
+    position: absolute;
   }
   video {
-    width: 100%
+    width: 100%;
   }
 }
 .audio-container {
   audio {
-    width: 100%
+    width: 100%;
   }
-}
+}

src/app/molecules/message/Message.jsx

@@ -610,6 +610,8 @@ function genMediaContent(mE) {
   let msgType = mE.getContent()?.msgtype;
   if (mE.getType() === 'm.sticker') msgType = 'm.sticker';
 
+  const blurhash = mContent?.info?.['xyz.amorgan.blurhash'];
+
   switch (msgType) {
     case 'm.file':
       return (
@@ -629,6 +631,7 @@ function genMediaContent(mE) {
           link={mx.mxcUrlToHttp(mediaMXC)}
           file={isEncryptedFile ? mContent.file : null}
           type={mContent.info?.mimetype}
+          blurhash={blurhash}
         />
       );
     case 'm.sticker':
@@ -666,6 +669,7 @@ function genMediaContent(mE) {
           height={typeof mContent.info?.h === 'number' ? mContent.info?.h : null}
           file={isEncryptedFile ? mContent.file : null}
           type={mContent.info?.mimetype}
+          blurhash={blurhash}
         />
       );
     default:

src/client/state/RoomsInput.js

@@ -3,12 +3,34 @@ import { micromark } from 'micromark';
 import { gfm, gfmHtml } from 'micromark-extension-gfm';
 import encrypt from 'browser-encrypt-attachment';
 import { math } from 'micromark-extension-math';
+import { encode } from 'blurhash';
 import { getShortcodeToEmoji } from '../../app/organisms/emoji-board/custom-emoji';
 import { mathExtensionHtml, spoilerExtension, spoilerExtensionHtml } from '../../util/markdown';
 import { getImageDimension } from '../../util/common';
 import cons from './cons';
 import settings from './settings';
 
+const blurhashField = 'xyz.amorgan.blurhash';
+
+function encodeBlurhash(img) {
+  const canvas = document.createElement('canvas');
+  canvas.width = 100;
+  canvas.height = 100;
+  const context = canvas.getContext('2d');
+  context.drawImage(img, 0, 0, canvas.width, canvas.height);
+  const data = context.getImageData(0, 0, canvas.width, canvas.height);
+  return encode(data.data, data.width, data.height, 4, 4);
+}
+
+function loadImage(url) {
+  return new Promise((resolve, reject) => {
+    const img = new Image();
+    img.onload = () => resolve(img);
+    img.onerror = (err) => reject(err);
+    img.src = url;
+  });
+}
+
 function loadVideo(videoFile) {
   return new Promise((resolve, reject) => {
     const video = document.createElement('video');
@@ -300,10 +322,11 @@ class RoomsInput extends EventEmitter {
     let uploadData = null;
 
     if (fileType === 'image') {
-      const imgDimension = await getImageDimension(file);
+      const img = await loadImage(URL.createObjectURL(file));
 
-      info.w = imgDimension.w;
-      info.h = imgDimension.h;
+      info.w = img.width;
+      info.h = img.height;
+      info[blurhashField] = encodeBlurhash(img);
 
       content.msgtype = 'm.image';
       content.body = file.name || 'Image';
@@ -313,8 +336,11 @@ class RoomsInput extends EventEmitter {
 
       try {
         const video = await loadVideo(file);
+
         info.w = video.videoWidth;
         info.h = video.videoHeight;
+        info[blurhashField] = encodeBlurhash(video);
+
         const thumbnailData = await getVideoThumbnail(video, video.videoWidth, video.videoHeight, 'image/jpeg');
         const thumbnailUploadData = await this.uploadFile(roomId, thumbnailData.thumbnail);
         info.thumbnail_info = thumbnailData.info;