Using bash scripts to automate #AVpres workflows

#!/bin/bash

# Create full timestamp variable for use in logging
date_FULL=$(date +'%Y-%m-%d  - %T')

# ===============================================================
# === Source and destination folder paths as variables ==========
# ===============================================================

# Variables for Isilon and BFI QNAP rawcooked folder
dpx_source="/mnt/isilon/"
mkv_destination_qnap="/mnt/qnap/Public/rawcooked/"

# Removes the temporary list again so cook runs from fresh list each time
rm ${mkv_destination_qnap}temporary_rawcook_list.txt
# Use temp_queued_list.txt to check before cooking a dpx folder, to make sure it has not been cooked, or is currently being cooked.
ls ${mkv_destination_qnap}mkv_cooked > ${mkv_destination_qnap}temp_queued_list.txt
# Create new temporary_rawcook_list.txt each time script runs
touch ${mkv_destination_qnap}temporary_rawcook_list.txt

# Write a START note to the logfile
echo "========== DPX rawcooking STARTED  ===================================================== $date_FULL" >> ${mkv_destination_qnap}rawcook.log

# Find all folders in the source path, then pass random 20 to GNU Parallel job to rawcook with --check (to check reversibility) -y (to suspend interaction)
# --accept-gaps (to allow for breaks in sequence numbering), -o path to MKV destination and output to log file

find ${dpx_source}dpx_to_cook/ -maxdepth 1 -type d -name "N*" | shuf -n 20 | while IFS= read -r folders; do
  folder_clean=$(echo "$folders" | cut -c 29-)
  count_cooked=$(cat ${mkv_destination_qnap}rawcooked_success.log | grep -c "$folder_clean")
  count_queued=$(grep -c "$folder_clean" ${mkv_destination_qnap}temp_queued_list.txt)
  echo "$folder_clean: count cooked = $count_cooked, count_queued = $count_queued" >> ${mkv_destination_qnap}rawcook.log
  if [ "$count_cooked" -eq 0 ] && [ "$count_queued" -eq 0 ];
   then
    echo "$date_FULL DPX folder will be cooked: $folders" >> ${mkv_destination_qnap}rawcook.log
    echo "$folder_clean" >> ${mkv_destination_qnap}temporary_rawcook_list.txt
   else
    echo "$date_FULL Skipping DPX folder, it is already cooked or being cooked: $folders" >> ${mkv_destination_qnap}rawcook.log 
  fi
done

# Sends the temporary rawcooked list to parallel to cook multiple RAWcook jobs 
cat ${mkv_destination_qnap}temporary_rawcook_list.txt | parallel --jobs 4 "rawcooked --check -y --accept-gaps /mnt/isilon/dpx_to_cook/{} -o ${mkv_destination_qnap}mkv_cooked/{}.mkv &>> ${mkv_destination_qnap}mkv_cooked/{}.mkv.txt"

# Write an END note to the logfile
echo "========== DPX rawcooking ENDED  =================================================== $date_FULL" >> ${mkv_destination_qnap}rawcook.log

#!/bin/bash

# Create full timestamp variable for use in logging below
date_FULL=$(date +'%Y-%m-%d  - %T')

# ===============================================================
# === Source and destination folder paths as variables ==========
# ===============================================================

# = Variables for Isilon and BFI QNAP rawcooked folder
dpx_source="/mnt/isilon/"
mkv_destination_qnap3="/mnt/qnap/Public/rawcooked/"
mkv_autoingest_qnap2="/mnt/qnap/Public/autoingest/"

#  Write a START note to the logfile
echo "========== Post-rawcook workflows STARTED  ================================================= $date_FULL" >> ${mkv_destination_qnap3}post_rawcooked.log

# ====================================================================
# FOR ====MEDIACONCH POLICY FAILURES==== Remove fails to Killed folder
# ====================================================================

# MediaConch policy checks for missing duration, bitrate >300Mb/s, slices >16 and many more.

find ${mkv_destination_qnap3}mkv_cooked/ -name "*.mkv" -mmin +20 | while IFS= read -r files; do
check=$(mediaconch --force -p /mnt/isilon/rawcooked/mkv_policy.xml "$files" | grep "fail")
filename=$(basename "$files") 
  if [ -z "$check" ];
    then
      echo "*** RAWcooked MKV file $filename has passed the Mediaconch policy. Whoopee ***" >> ${mkv_destination_qnap3}post_rawcooked.log
    else
      {
        echo "*** FAILED RAWcooked MKV $filename has failed the mediaconch policy. Grrrrr ***"
        echo "*** Moving $filename to killed directory, and amending log fail_${filename}.txt ***"
        echo "$check"
      } >> ${mkv_destination_qnap3}post_rawcooked.log
      echo "$filename" > ${mkv_destination_qnap3}temp_mediaconch_policy_fails.txt
  fi
done

grep ^N ${mkv_destination_qnap3}temp_mediaconch_policy_fails.txt | parallel --progress --jobs 10 "mv ${mkv_destination_qnap3}mkv_cooked/{} ${mkv_destination_qnap3}killed/{}"
grep ^N ${mkv_destination_qnap3}temp_mediaconch_policy_fails.txt | parallel --progress --jobs 10 "mv ${mkv_destination_qnap3}mkv_cooked/{}.txt ${mkv_destination_qnap3}logs/fail_{}.txt"

# =============================================================
# FOR ====PASS==== - i.e. successful RAWcooked --check outcomes
# =============================================================

# Check for presence of PASS cases, process those, and add to log files
count=$(grep -c "Reversability was checked, no issue detected" ${mkv_destination_qnap3}mkv_cooked/*.mkv.txt | grep -c "1")
if [ "${count}" -gt 0 ];
 then
   # Check in the .mkv.txt files for the -no issue detected- rows and output them to the rawcooked_success.log
   grep -l "Reversability was checked, no issue detected" ${mkv_destination_qnap3}mkv_cooked/*.mkv.txt | rev | cut -c 9- | rev >> ${mkv_destination_qnap3}rawcooked_success.log

   # Log the moves of MKV into autoingest
   echo "*** No reversibility issues detected with these Matroska files, moving them into autoingest ***" >> ${mkv_destination_qnap3}post_rawcooked.log
   grep -l "Reversability was checked, no issue detected" ${mkv_destination_qnap3}mkv_cooked/*.mkv.txt | rev | cut -c 9- | rev >> ${mkv_destination_qnap3}post_rawcooked.log
 
   # Move the mkv into ingest workflow
   grep -l "Reversability was checked, no issue detected" ${mkv_destination_qnap3}mkv_cooked/*.mkv.txt | rev | cut -c 5- | rev | cut -c 55- | parallel --progress --jobs 10 "mv ${mkv_destination_qnap3}mkv_cooked/{} ${mkv_autoingest_qnap2}{}" 
 
   # Log the dpx folders that are being moved
   echo "*** No reversibility issues detected with these DPX sequences, moving the folders into /mnt/isilon/dpx_cooked/ ***" >> ${mkv_destination_qnap3}post_rawcooked.log
  
   grep -l "Reversability was checked, no issue detected" ${mkv_destination_qnap3}mkv_cooked/*.mkv.txt | rev | cut -c 9- | rev >> ${mkv_destination_qnap3}post_rawcooked.log
 
   # Move the dpx folders from dpx_to_cook folder into dpx_cooked, otherwise they will be cooked again on the next pass.
   grep -l "Reversability was checked, no issue detected" ${mkv_destination_qnap3}mkv_cooked/*.mkv.txt | rev | cut -c 9- | rev | cut -c 55- | parallel --progress --jobs 10 "mv ${dpx_source}dpx_to_cook/{} ${dpx_source}dpx_cooked/{}"
 
   # Move the txt files to logs folder
   grep -l "Reversability was checked, no issue detected" ${mkv_destination_qnap3}mkv_cooked/*.mkv.txt | cut -c 55- | parallel --progress --jobs 10 "mv ${mkv_destination_qnap3}mkv_cooked/{} ${mkv_destination_qnap3}logs/{}" 
 
 else
   # Output 'no passes to deal with' message to log
   echo "*** No successful Matroska files to move into autoingest this time ***" >> ${mkv_destination_qnap3}post_rawcooked.log
fi

# ===============================================================
# FOR ====FAIL==== - i.e. unsuccessful RAWcooked --check outcomes
# ===============================================================

# Check for presence of FAIL cases, process those, and add to log files
count=$(grep -c "Reversability was checked, issues detected\|Error: undecodable file (can not be open)\|Error: undecodable cannot open file for reading\|Conversion failed!\|Error: unsupported DPX Number of image elements\|Error: untested multiple slices counts" ${mkv_destination_qnap3}mkv_cooked/*.mkv.txt | grep -v -c ':0')
if [ "${count}" -gt 0 ];
 then
   echo "*** Reversibility or encoding issues were detected with these Matroska files, deleting them to allow RAWcooked to try again ***" >> ${mkv_destination_qnap3}post_rawcooked.log	
   grep -l "Reversability was checked, issues detected\|Error: undecodable file (can not be open)\|Error: undecodable cannot open file for reading\|Conversion failed!\|Error: unsupported DPX Number of image elements\|Error: untested multiple slices counts" ${mkv_destination_qnap3}mkv_cooked/*.mkv.txt | rev | cut -c 5- | rev >> ${mkv_destination_qnap3}post_rawcooked.log
	
   # Delete the Matroskas
   grep -l "Reversability was checked, issues detected\|Error: undecodable file (can not be open)\|Error: undecodable cannot open file for reading\|Conversion failed!\|Error: unsupported DPX Number of image elements\|Error: untested multiple slices counts" ${mkv_destination_qnap3}mkv_cooked/*.mkv.txt | rev | cut -c 5- | rev | parallel --progress --jobs 10 "rm {}"
	
   # Move the txt files to logs folder and prepend -fail- to filename
   grep -l "Reversability was checked, issues detected\|Error: undecodable file (can not be open)\|Error: undecodable cannot open file for reading\|Conversion failed!\|Error: unsupported DPX Number of image elements\|Error: untested multiple slices counts" ${mkv_destination_qnap3}mkv_cooked/*.mkv.txt | cut -c 55- | parallel --progress --jobs 10 "mv ${mkv_destination_qnap3}mkv_cooked/{} ${mkv_destination_qnap3}logs/fail_{}"
   
 else
   # Output 'no fails to deal with' message to log
   echo "*** No failed Matroska files to delete this time ***" >> ${mkv_destination_qnap3}post_rawcooked.log
fi

# ===============================================================
# FOR ==== INCOMPLETE ==== - i.e. killed processes ==============
# ===============================================================

# This block manages the remaining INCOMPLETE cooks not removed by the MediaConch policy
# It uses find mmin +1440 to find txt files not modified in last 1440 minutes, then deletes the text file, and associated reversibility and mkv files

# Check for presence of stale .mkv.txt files remaining after processes above, using mmin to query modified date minutes > 1440
count_old=$(find ${mkv_destination_qnap3}mkv_cooked/ -name "*.mkv.txt" -mmin +1440 -size +10k | grep -c ".mkv.txt")
if [ "${count_old}" -gt 0 ];
 then
   echo "*** Files found indicating stalled or killed process, deleting them to enable reprocessing ***" >> ${mkv_destination_qnap3}post_rawcooked.log	
   
   # Add to log file
   find ${mkv_destination_qnap3}mkv_cooked/ -name "*.mkv.txt" -mmin +1440 -size +10k | rev | cut -c 9- | rev >> ${mkv_destination_qnap3}post_rawcooked.log
   
   # Delete any MKVs that exist alongside these stalled text files
   find ${mkv_destination_qnap3}mkv_cooked/ -name "*.mkv.txt" -mmin +1440 -size +10k | rev | cut -c 5- | rev | parallel --jobs 10 "rm {}*"

   # Delete the files
   find ${mkv_destination_qnap3}mkv_cooked/ -name "*.mkv.txt" -mmin +1440 -size +10k | parallel --jobs 10 "rm {}*"
	
 else
   echo "*** No incomplete files to delete this time ***" >> ${mkv_destination_qnap3}post_rawcooked.log
fi

# Write an END note to the logfile 
echo "========== Post-rawcook workflows ENDED  =================================================== $date_FULL" >> ${mkv_destination_qnap3}post_rawcooked.log

# Update the count of successful cooks at top of the success log  
# First create new temp_success_log with timestamp
echo "==== Updated $date_FULL =====================================" > ${mkv_destination_qnap3}temp_rawcooked_success.log

# Count lines in success_log and create count variable, output that count to new success log, then output all lines with /mnt* to the new log
grep "/mnt/" ${mkv_destination_qnap3}rawcooked_success.log >> ${mkv_destination_qnap3}temp_rawcooked_success.log
success_count=$(grep -c "/mnt/" ${mkv_destination_qnap3}temp_rawcooked_success.log)
echo "==== Successful cooks: $success_count ==============================================" >> ${mkv_destination_qnap3}temp_rawcooked_success.log

# Sort the log and remove any non-unique lines
sort ${mkv_destination_qnap3}temp_rawcooked_success.log | uniq | sort -r > ${mkv_destination_qnap3}temp_rawcooked_success_unique.log
  
# Move the new log renaming it to overwrite the old log
mv ${mkv_destination_qnap3}temp_rawcooked_success_unique.log ${mkv_destination_qnap3}rawcooked_success.log