Synology Index Status

DATABASE STATUS on (2017-10-30 16:07:34)
    media    | count  |                                                               lastprocessed                                                           
 Music       |   2461 | /volume1/music/SomeMusicFile.mp3
 Photos      |    762 | /volume1/photo/SomePhoto.jpg
 Videos      |     58 | /volume1/video/SomeVideo.mp4
 Directories |    392 | /volume1/photo/SomeDir
    media    |        date         |        mdate
 Music       | 2017-02-05 01:48:37 | 2016-10-22 22:15:24
 Photos      | 2009-01-06 21:34:54 | 2009-01-06 21:34:54
 Videos      | 2014-03-08 21:00:40 | 2012-08-21 13:22:34
 Directories | 2015-01-09 15:30:41 | 2015-01-09 15:46:53

Showing real time file access of indexing/thumbnailing processes (5 sec interval, so some lines are missed):

QUEUE STATUS
Indexing queue             53
Photo conversion progress: 1103/1475 (75%), thumbnails: 2190/2934 (75%)
Video conversion progress: 0/0
CURRENT PROCESSES
postgresphoto     20.3  SELECT
synoindexplugind  3.1
mediaparser
synoindexd
synothumb         0.0   /volume1/photo/SomePhoto.jpg
synomkthumbd
ffmpeg-thumb
frecognition      25.0  /volume1/photo/SomePhoto.jpg
UniversalSearch/fileindexd         0    
UniversalSearch/synoelasticd       0
LAST DETECTED
mediaparser       NOTimplemented
synoindexd        NOTimplemented
indexworkerd      NOTimplemented
synothumb         /volume1/photo/SomePhoto.jpg
synomkthumbd      /volume1/photo/SomePhoto.jpg
ffmpegthumb
facerecognition   /volume1/photo/SomePhoto.jpg
fileindexd         

#!/usr/bin/bash
#
# Nice Index Status
# version 1.1
#
# Last updated 2019-10-06 by RoBo

# ToDo:
# on nas within busybox
#cat: /var/spool/conv_progress_photo: No such file or directory
#cat: /var/spool/conv_progress_photo: No such file or directory
#awk: cmd. line:1: fatal: division by zero attempted
#./niceindexstatus: line 200: procopenfiles: command not found

# some info
#
# PHOTO
# Filename Max width or height (pixels)
# SYNOPHOTO_THUMB_B.jpg 640
# SYNOPHOTO_THUMB_M.jpg 320
# SYNOPHOTO_THUMB_PREVIEW.jpg 160
# SYNOPHOTO_THUMB_S.jpg 120
# SYNOPHOTO_THUMB_XL.jpg 1.280
#
# VIDEO
# Filename Max width or height (pixels)
# SYNOPHOTO:THUMB_M.jpg 320
# SYNOPHOTO:THUMB_XL.jpg video original
# SYNOPHOTO:VIDEO_SCREENSHOT.jpg video original
#
# The short dimension of the thumbnail is calculated with the aspect ratio of the original photo or video. If the video is rotated (metadata Rotation=xx), for instance by using an iPhone, the thumbnails are also rotated and displayed as expected. Metadata are not saved to the thumbnails.

# The cover pictures for an album are located in the @eaDir folder at the same level as the album folder. Inside this folder are subfolders with the same album names, containing a file SYNOPHOTO:ALBUM.cover. It’s a text file with the filename of the cover photo, nothing else. If no cover photo is defined, the first image in the album is used as default cover. If a movie is selected as a cover image, the filename of the video is listed in the SYNOPHOTO:ALBUM.cover text file.

#A second file SYNOPHOTO:ALBUM.sort is in the same @eaDir folder. A content example is shown below :
#
#{"type":"3","order":"0","list":["IMG0001.jpg","IMG0002.jpg","IMG0003.jpg"]}



#### To Add
#root 21846 1 4 Nov08 ? 13:57:52 /var/packages/SynoFinder/target/sbin/synoelasticd
#root 21864 1 0 Nov08 ? 00:11:29 /var/packages/SynoFinder/target/sbin/fileindexd


conv_progress_photo=/var/spool/conv_progress_photo
conv_progress_video=/var/spool/conv_progress_video
indexingqueue=/var/spool/syno_indexing_queue
WaitTime=5
CPUcores=$(grep -c ^processor /proc/cpuinfo)

COLOR_WHITE="\033[01;39m"
COLOR_RES="\033[0m"


Arguments=()
while [[ $# -gt 0 ]]
do
key="$1"

case $key in
-w|-wait|--wait)
shift
WaitTime="$1"
shift
;;
-h|--help)
echo "Usage: niceindexstatus [option]"
echo ""
echo "Purpose: Provide an overview about the indexing status."
echo "Attention: This script has to be executed as root."
echo ""
echo "Options:"
echo "-w|--wait Set the wait time in seconds, default= 5s"
echo
exit
;;
*) # unknown option
Arguments+=("$1") # save it in an array for later
shift # past argument
;;
esac
done
set -- "${Arguments[@]}" # restore positional parameters

# if there is no list of files, then create '.' list file
#files=("$@")
#nroffiles=${#files[*]}
#c="1"
#if [ "$nroffiles" -eq "0" ] ; then
# for entry in .
# do
# files[$((c-1))]="$entry"
# ((c++))
# done
#fi


if [[ $EUID -ne 0 ]]; then
echo "This script needs to be run as root"
exit 1
fi


############################################################

clear
# function doesn't seem to work
#function procgrep param {
# cat param | egrep -v '\/dev\/null|socket:|anon_inode:\[eventpoll\]|total 0|ls: cannot access \/proc\/' | cut -d\> -f2-
#}

echo -e "${COLOR_WHITE}DATABASE STATUS$COLOR_RES on ($(date '+%Y-%m-%d %H:%M:%S'))"

psql mediaserver postgres -c "SELECT 'Music' AS Media, Nr.Count, Path AS LastProcessed FROM music, (SELECT COUNT(*) AS Count FROM music) AS Nr WHERE id=(SELECT MAX(id) FROM music)
UNION ALL
SELECT 'Photos' AS Media, Nr.Count, Path AS LastProcessed FROM photo, (SELECT COUNT(*) AS Count FROM photo) AS Nr WHERE id=(SELECT MAX(id) FROM photo)
UNION all
SELECT 'Videos' as Media, Nr.Count, Path AS LastProcessed FROM video, (SELECT COUNT(*) as Count FROM video) as Nr WHERE id=(SELECT MAX(id) FROM video)
UNION all
SELECT 'Directories' as Media, Nr.Count, Path AS LastProcessed FROM directory, (SELECT COUNT(*) as Count FROM directory) AS Nr WHERE id=(SELECT MAX(id) FROM directory);" |\
egrep -v '^$| rows)|-------------'

psql mediaserver postgres -c "SELECT 'Music' AS Media, Date, MDate FROM music, (SELECT COUNT(*) AS Count FROM music) AS Nr WHERE id=(SELECT MAX(id) FROM music)
UNION ALL
SELECT 'Photos' AS Media, Date, MDate FROM photo, (SELECT COUNT(*) AS Count FROM photo) AS Nr WHERE id=(SELECT MAX(id) FROM photo)
UNION all
SELECT 'Videos' as Media, Date, MDate FROM video, (SELECT COUNT(*) as Count FROM video) as Nr WHERE id=(SELECT MAX(id) FROM video)
UNION all
SELECT 'Directories' as Media, Date, MDate FROM directory, (SELECT COUNT(*) as Count FROM directory) AS Nr WHERE id=(SELECT MAX(id) FROM directory);" | egrep -v ' rows)|-------------'

echo "Showing real time file access of indexing/thumbnailing processes ($WaitTime sec interval, so some lines are missed):"
echo -e "\033[s" # save the cursor position

while true;
do
# echo -e "No open files of indexing/thumnail processes found\033[u"

echo -e "${COLOR_WHITE}QUEUE STATUS$COLOR_RES\033[K"

if [ -f "$indexingqueue" ] ; then
total=$(cat "$indexingqueue" | wc -l)
else
total="0"
fi
echo -e "Indexing queue $total \033[K"


comp=$(cat "$conv_progress_photo"|grep 'completed='|cut -f2 -d"=")
total=$(cat "$conv_progress_photo"|grep 'total='|cut -f2 -d"=")
if [ "$total" != "0" ]; then
perc=" ($(awk -v comp=$comp -v total=$total 'BEGIN{printf("%.0lf", comp/total*100)}')%)"
else
perc=""
fi
echo -en "Photo conversion progress: $comp/$total$perc"

comp=$(cat "$conv_progress_photo"|grep 'completed_thumb='|cut -f2 -d"=")
total=$(cat "$conv_progress_photo"|grep 'total_thumb='|cut -f2 -d"=")
if [ "$total" != "0" ]; then
perc="($(awk -v comp=$comp -v total=$total 'BEGIN{printf("%.0lf", comp/total*100)}')%)"
else
perc=""
fi
echo -e ", thumbnails: $comp/$total $perc\033[K"

comp=$(cat "$conv_progress_video"|grep 'completed='|cut -f2 -d"=")
total=$(cat "$conv_progress_video"|grep 'total='|cut -f2 -d"=")
if [ "$total" != "0" ]; then
perc="($(awk -v comp=$comp -v total=$total 'BEGIN{printf("%.0lf", comp/total*100)}')%)"
else
perc=""
fi
echo -e "Video conversion progress: $comp/$total $perc\033[K"
echo -en "\033[K"


echo -e "${COLOR_WHITE}CURRENT PROCESSES$COLOR_RES\033[K"



# database queries
postgresphoto=$(ps -ef 2>/dev/null | grep 'postgres photo \[local\]' | grep -v grep | awk '{print $12}'|sort -r|head -n 1)
if [ "$cpu_facerecognition" == "%CPU" ]; then
cpu_facerecognition="" # the process was already killed
fi

if [ "$postgresphoto" != "" ]; then
cpu_postgresphoto=$(echo "$(top -b -p $(ps -ef 2>/dev/null | grep 'postgres photo \[local\]' | grep -v grep | awk '{print $2}'|sort -r|head -n 1) -n 1 | tail -n 1 | awk -v cpus=$CPUcores '{printf("%2.f\n", $7/cpus)}')")
echo -e "postgresphoto $cpu_postgresphoto \t$postgresphoto\033[K"
else
echo -e "postgresphoto \033[K"
fi


# synoindexplugind - checks if there is something changed I guess
synoindexplugind=$(ps -ef 2>/dev/null| grep synoindexplugind | grep -v grep | awk '{print $2}')
if [ "$synoindexplugind" != "" ]; then
cpu_synoindexplugind=$(echo "$(top -b -p $(ps -ef 2>/dev/null| grep synoindexplugind | grep -v grep | awk '{print $2}') -n 1 | tail -n 1 | awk -v cpus=$CPUcores '{printf("%2.f\n", $7/cpus)}')")
echo -e "synoindexplugind $cpu_synoindexplugind \033[K"
else
echo -e "synoindexplugind \033[K"
fi



# mediaparcer (not sure what it does)
# if procopenfiles synomediaparser -q; then
mediaparser=$(procopenfiles synomediaparser)
if [ "$mediaparser" != "" ]; then
cpu_mediaparser=$(echo "$(top -b -p $(ps -ef 2>/dev/null| grep synomediaparser | grep -v grep | awk '{print $2}') -n 1 | tail -n 1 | awk -v cpus=$CPUcores '{printf("%2.f\n", $7/cpus)}')")
echo -e "mediaparser $cpu_mediaparser \t$synomediaparser \033[K"
else
echo -e "mediaparser \033[K"
fi
# fi

# indexers
synoindexd=$(procopenfiles synoindexd)
if [ "$synoindexd" != "" ]; then
cpu_synoindexd=$(echo "$(top -b -p $(ps -ef 2>/dev/null| grep synoindexd | grep -v grep | awk '{print $2}') -n 1 | tail -n 1 | awk -v cpus=$CPUcores '{printf("%2.f\n", $7/cpus)}')")
echo -e "synoindexd $cpu_synoindexd \t$synoindexd \033[K"
else
echo -e "synoindexd \033[K"
fi

# cpu_indexdworkerd="XXXX"
# echo -e "\033[01;39m[indexdworkerd]\033[0m $cpu_indexdworkerd\t$(procopenfiles synoindexworkerd) \033[K"

# Thumbnails
# echo -e "\033[01;39m[synothumb]\033[0m " $(procopenfiles synothumb) \033[K"
pid=$(ps -ef | grep synothumb | grep -v grep | awk '{print $2}')
if [ "$pid" != "" ]; then
synothumb=$(procopenfiles synothumb)
cpu_synothumb=$(echo "$(top -b -p $pid -n 1 2>/dev/null| tail -n 1 | awk -v cpus=$CPUcores '{printf("%2.f\n", $7/cpus)}')")
echo -e "synothumb $cpu_synothumb \t$synothumb \033[K"
lastsynothumb="$synothumb"
else
echo -e "synothumb \033[K"
fi

synomkthumbd=$(procopenfiles synomkthumbd)
if [ "$synomkthumbd" != "" ]; then
cpu_synomkthumbd=" "
echo -e "synomkthumbd $cpu_synomkthumbd \t$(procopenfiles synomkthumbd) \033[K"

if [[ ${synomkthumbd} != *"/var/spool/"* ]]; then
lastsynomkthumbd="$synomkthumbd"
fi
else
echo -e "synomkthumbd \033[K"
fi


pid=$(ps -ef | grep ffmpeg-thumb | grep -v grep | awk '{print $2}')
# echo "*** pid: '$pid'"
if [ "$pid" != "" ]; then
ffmpegthumb=$(procopenfiles ffmpeg-thumb)
cpu_ffmpegthumb=$(echo "$(top -b -p $pid -n 1 2>/dev/null| tail -n 1 | awk -v cpus=$CPUcores '{printf("%2.f\n", $7/cpus)}')")
echo -e "ffmpeg-thumb $cpu_ffmpegthumb \t$ffmpegthumb \033[K"
lastffmpegthumb="$ffmpegthumb"
else
echo -e "ffmpeg-thumb \033[K"
fi

# Others
# if [ "$(ps -ef | grep facerecognition | grep -v grep | sed 's/^.* -F //')" != "" ]; then
### echo -e "*** Starting facerecognition part \033[K"
string=$(ps -ef | grep facerecognition | grep -v grep)
pid=$(echo $string | awk '{print $2}')
# get the file facerecognition is working on:
facerecognition=$(echo $string | sed 's/^.* -F //')
### echo "*** pid: '$pid', facerec: '$facerecognition'"
if [ "$facerecognition" != "" ]; then
cpu_facerecognition=$(top -b -p $pid -n 1 | tail -n 1 | awk '{print $7}')
if [ "$cpu_facerecognition" == "%CPU" ]; then
cpu_facerecognition="" # the process was already killed
fi
### echo -e "*** cpu_facerecognition: $cpu_facerecognition\033[K"
if [ "$cpu_facerecognition" != "" ]; then
# cpu_facerecognition=$(echo "print round($(echo $cpu_facerecognition) / 4,1)" | python 2>/dev/nul)
cpu_facerecognition=$(gawk -v var=$cpu_facerecognition -v cpus=$CPUcores 'BEGIN {printf("%2.f", var/cpus)}')
else
cpu_facerecognition=" ? "
fi
echo -e "frecognition $cpu_facerecognition \t$facerecognition \033[K"
lastfacerecognition="$facerecognition"
else
echo -e "frecognition \033[K"
fi




pid=$(ps -ef | grep fileindexd | grep -v grep | awk '{print $2}')
if [ "$pid" != "" ]; then
if [ -f "/var/log/fileindexd.log" ]; then
fileindexd=$(tail -10 /var/log/fileindexd.log | grep 'IndexAttr: ' | tail -1 | cut -d\ -f6-)
else
# The /var/log/fileindexd.log file does not exist, so we can't provide any details
fileindexd="no details as /var/log/fileindexd.log does not exist"
fi
cpu_fileindexd=$(echo "$(top -b -p $pid -n 1 2>/dev/null| tail -n 1 | awk -v cpus=$CPUcores '{printf("%2.f\n", $7/cpus)}')")
echo -e "UniversalSearch/fileindexd $cpu_fileindexd \t$fileindexd \033[K"
lastfileindexd="$fileindexd"
else
echo -e "UniversalSearch/fileindexd \033[K"
fi



pid=$(ps -ef | grep synoelasticd | grep -v grep | awk '{print $2}')
if [ "$pid" != "" ]; then
if [ -f "/var/log/fileindexd.log" ]; then
# following line needs to be tested/confirmed that it's working:
synoelasticd=$(tail -10 /var/log/fileindexd.log | grep 'IndexAttr: ' | tail -1 | cut -d\ -f6-)
else
synoelasticd="no details as /var/log/fileindexd.log does not exist"
fi
cpu_synoelasticd=$(echo "$(top -b -p $pid -n 1 2>/dev/null| tail -n 1 | awk -v cpus=$CPUcores '{printf("%2.f\n", $7/cpus)}')")
echo -e "UniversalSearch/synoelasticd $cpu_synoelasticd \033[K"
#lastfileindexd="$fileindexd"
else
echo -e "UniversalSearch/synoelasticd \033[K"
fi




#echo -e "\e[01;39m[spoolfiles]\e[0m photo: $(ls -l /var/spool/conv_progress_photo* | wc -l), video: $(ls -l /var/spool/conv_progress_video* | wc -l) \e[K"

echo -en "\033[K"
echo -e "${COLOR_WHITE}LAST DETECTED$COLOR_RES\033[K"

echo -e "mediaparser ${synomediaparser}NOTimplemented\033[K"
echo -e "synoindexd ${synoindexd}NOTimplemented\033[K"
echo -e "indexworkerd ${synoindexworkerd}NOTimplemented\033[K"
echo -e "synothumb $lastsynothumb \033[K"
echo -e "synomkthumbd $lastsynomkthumbd \033[K"
echo -e "ffmpegthumb $lastffmpegthumb \033[K"
echo -e "facerecognition $lastfacerecognition \033[K"
echo -e "fileindexd $lastfileindexd \033[K"

echo -e "\033[K\033[1B\033[K\033[1B\033[K\033[1B\033[K\033[1B\033[K\033[1B\033[K" #move cursor down 1 line and clear until end of line and do that a few times

echo -e "\033[u" # move cursor to stored position

sleep $WaitTime
done