From d680d52425f7666d40b633d03dcefa02ba30694d Mon Sep 17 00:00:00 2001 From: Vincent Riquer Date: Fri, 20 Feb 2026 04:07:38 +0100 Subject: [PATCH] Comment atom --- atom | 136 +++++++++++++++++++++++++++++++++++++++++++++++++++++++++-- 1 file changed, 133 insertions(+), 3 deletions(-) diff --git a/atom b/atom index b8b2e6a..cac925d 100755 --- a/atom +++ b/atom @@ -13,6 +13,9 @@ # A copy of the GNU General Public License v3 is includded in the LICENSE file # at the root of the project. +# Directories for various required data. Set by the configure script and the +# Makefile. +# Also save $IFS just in case. declare -r \ DOCDIR=%DOCDIR% \ LIBDIR=%LIBDIR% \ @@ -26,7 +29,7 @@ declare -r \ ## Define exit codes source "$SHAREDIR"/errorcodes -# config structures +# Config structures. declare -A \ destinationenabled \ destinationascii \ @@ -51,10 +54,13 @@ declare -A \ exit $EBASHVERS } +# Locales break parsingg. LC_ALL=C +# Enable extended globbing for some filename manipulations. shopt -s extglob +# Array of ID3v1 genres, number to name mapping. source "$SHAREDIR"/id3genres for function in "$LIBDIR"/*/* @@ -62,6 +68,7 @@ do source "$function" done +# Migrate old config to XDG where required if ! [[ -f "${XDG_CONFIG_HOME:-$HOME/.config}/AtOM/atom.cfg" ]] \ && [[ -f "$HOME/.atom/atom.cfg" ]] then @@ -141,6 +148,9 @@ do done askconf() { + # Prompt user to interactively create a config if it doesn't exist and + # we're not in cron mode. + # Called recursively until a valid answer is given. if (( cron )) then echo 'Non-interactive, not running setup. Please run atom -S.' >&2 @@ -165,6 +175,7 @@ askconf() { esac } +# Read config if it exists, call askconf otherwise. if [ ! -f "$cffile" ] then if [ ! -d "${cffile%/*}" ] @@ -176,14 +187,17 @@ then fi getConfig +# Uf user wants to changeg config, run setup. (( forcesetup )) && setup +# Deactivate `!` history expansion, just in case. set +H # Apply CLI overrides [ -n "$cliload" ] && maxload=$cliload [ -n "$cliltimer" ] && loadinterval=$cliltimer +# Print config if requested or in debug mode. Exit if only dumping config. (( debug || cfgdump )) && printConfig (( cfgdump )) && exit @@ -191,6 +205,11 @@ set +H sanityCheck openDatabase +# create missing destination directories and DB entries, get destination IDs +# for later use +createDestinations + +# Apply destinations "enabled" status in DB with respect to config. for destination in "${destinations[@]}" do if (( ${destinationenabled["$destination"]} )) @@ -201,14 +220,20 @@ do fi done -createDestinations - +# get source files. Update DB. getFiles +# Scan mime-types (for new/changed files). Update DB updateMimes +# Remove source files that are gone from DB. (`last_seen` column). +# FOREIGN KEY `source_file_id` on table `destination_files` gets set to NULL +# by `ON DELETE` parameter. We can use that to find destination files that need +# to be removed. removeObsoleteFiles +# remove destination files for which the source file is gone. (`source_file_id` +# NUUL -- `rm` them if they exist, remove the DB entry in any case) (( cron )) || echo -n 'Gathering files for cleaning...' echo ' SELECT COUNT(id) @@ -216,6 +241,7 @@ echo ' WHERE source_file_id is NULL;' >&3 read -u4 -r -d $'\0' removecount +# Gather in 500 files batches to avoid pipe overflow. until (( ${#removefile[@]} == removecount )) do echo ' @@ -247,6 +273,10 @@ done unset deleted unset removed echo 'BEGIN TRANSACTION;' >&3 + +# Remove the files if they exist. Unconditionnally remove from DB. +# Run in transactrion to speed up the process, COMMIT every 1000th file in case +# process gets killed. for id in ${!removefile[@]} do filename=${removefile[id]} @@ -274,8 +304,14 @@ echo -n "${deleted+$deleted files deleted${removed:+, }}${removed:+$removed remo (( deleted || removed )) && echo unset removecount deleted removed removefile +# Update tags for new/changed files and updated tag parsers. Update DB. +# Uses `tags.last_changge` vs `source_files.last_change` +# + `tags.tagreader` vs tagreader versions declared in running version's +# source code. updateTags +# Reset timestamps for files in destinations that were requested to be fully +# rebuilt. for forcedest in "${forceall[@]}" do if forcedestid=$(Select destinations id <<<"name = $forcedest") @@ -290,6 +326,18 @@ do fi done +# Create TEMPORARY (in-memory) tables for tasks. Up to 60 arguments per task +# (including command). +# +# `requires` designates the id of a tasks that must be completed before this +# one can be run. +# `required_by` is a counter of tasks that require this one. Used for temp +# files cleanup. +# `status` is 0 for pending tasks, 2 for failed tasks, 4 for completed tasks +# depended upon (temp file should be left intact). +# +# TRIGGER `fail_depends` sets status of all tasks that depend on a failed task +# to 2 echo ' CREATE TEMPORARY TABLE tasks( id INTEGER PRIMARY KEY, @@ -383,6 +431,7 @@ echo ' END; ' >&3 +# Get number of files to process. Apply `maxbatch` limit if specified. echo ' SELECT COUNT(source_files.id) FROM source_files @@ -406,6 +455,8 @@ then (( togo = filecount - maxbatch )) filecount=$maxbatch fi + +# Get files to process. Apply `maxbatch` limit if specified. echo ' SELECT source_files.id, @@ -452,16 +503,26 @@ echo '; read -u4 -r -d $'\0' line while ! [[ $line = AtOM:NoMoreFiles ]] do + # Append `::AtOM:SQL:Sep::` at the end of the line to make sure we can + # parse empty fields. decodefiles+=("$line::AtOM:SQL:Sep::") read -u4 -r -d $'\0' line done (( cron )) || echo -n $'Creating tasks...\033[K' +# Spawn perl coprocess for unicode to ascii conversion if needed. (( textunidecodeneeded )) && ascii +# Generate tasks for each file. Tasks that depend on other tasks (e.g. encoding +# depends on decoding) get the ID of the task they depend on in `requires` +# column. This is used to make sure that encoding doesn't start before decoding +# and other transforms have completed. echo 'BEGIN TRANSACTION;' >&3 for line in "${decodefiles[@]}" do + # Parsing SQL output is fun. We use `::AtOM:SQL:Sep::` as separator + # between fields at the end of the line to make sure we can parse empty + # fields. fileid=${line%%::AtOM:SQL:Sep::*} rest=${line#*::AtOM:SQL:Sep::} filename=${rest%%::AtOM:SQL:Sep::*} @@ -506,14 +567,24 @@ do rest=${rest#*::AtOM:SQL:Sep::} year=${rest%%::AtOM:SQL:Sep::*} unset rest + + # Skip destinations with formats for which tools are missing. case ${destinationformat["$destination"]} in vorbis) (( disableoggenc )) && continue ;; opus) (( disableopusenc )) && continue ;; mp3) (( disablelame )) && continue ;; esac + + # Create decoding task depending on mimetype. decodeFile + + # Build target directory path from source file path OR from rename + # pattern if set. getDestDir + # Same for filename. getDestFile + + # Set copied to 1 for files with extension in `copy_extension`. for copy_ext in "${destinationcopyext[@]}" do if [[ $filename =~ '.*\.'"$copy_ext"'$' ]] @@ -524,10 +595,17 @@ do done if (( copied )) then + # Copy file as-is to destination. copyFiles_matching else + # Call suitable function to create encoding task depending on + # destination format. + # encodeFile::mp3 + # encodeFile::opus + # encodeFile::vorbis encodeFile::${destinationformat[$destination]} fi + # Cleanup variables. Avoids leaking data between iterations. unset \ album \ albumartist \ @@ -570,6 +648,13 @@ echo 'COMMIT;' >&3 # remove perl unicode to ascii coprocess (( textunidecodeneeded )) && eval exec "${toascii[1]}>&-" +# Main loop. Run up to `concurrency` tasks in parallel, depending on system +# load. Spawn new tasks as old ones complete. Update progress info and commit +# DB every minute. +# +# Start with concurrency = maxload / 2, which seems like a good starting point +# on most systems. If result is 0, force to 1 to make sure we get going. If +# `-f ` option was used, use that value instead and don't change it. concurrency=$(( maxload / 2 )) (( concurrency )) || concurrency=1 active=0 @@ -590,6 +675,13 @@ do fi read humanload garbage < /proc/loadavg load=${humanload%.*} + # If `-f ` option was used, keep concurrency fixed to that + # value. Otherwise, adjust concurrency according to load and `maxload` + # value. If load is above `maxload`, reduce concurrency by 1 (down to 0 + # if `allow_zero_running` is set). If load is below `maxload`, increase + # concurrency by 1 (only if all slots are populated). + # Don't update concurrency more often than every `load-interval` seconds + # to reduce histeresis. if (( fixed_workers )) then concurrency="$fixed_workers" @@ -609,9 +701,30 @@ do fi fi fi + # check if workers have finished. + # If a worker finished with non-zero exit code, mark the task as failed + # in DB. TRIGGER `fail_depends` will fail all tasks that depend on it. checkworkers + # If task failed, set status to 2, remove temp files if any and + # increment `failed` counter. Don't delete task from DB to keep track + # of failed tasks for final report. + # If task was successful, update destination_files.last_change to + # source_files.last_change, update old_filename to previous + # (destination file) filename, store rename_pattern, fat32compat and + # ascii settings. + # Set status to 4 for tasks that were depended upon by other tasks to + # avoid cleaning up temp files before all tasks depending on them have + # completed, delete task otherwise. cleaner + # Look for pending tasks that can be started (required tasks's status + # is 4 or reqquires is NULL) and start them. + # Trigger a dump of the tasks table if it's inconsistent (no running + # tasks, no ready tasks, but pending tasks exist) and exit with error + # $ETASKLEFT. master + + # "Fancy" progress info. Only calculate if at least one task has + # succeeded to avoid division by zero. if (( ran - failed )) then currenttime=$timestamp @@ -650,6 +763,7 @@ do fmtworkers='W:%i/%i' fmtprogress="T:%${#taskcount}i/%i (F:%i) %3i%%" fmttime='%2id %2ih%02im%02is (A:%4.1fs/task)' + # Abuse timeformatting to get ETA. eta="ETA:$( printf "%(%c)T" "$(( currenttime + secsremaining ))" )" @@ -668,6 +782,7 @@ do ${minutes:-0} \ ${seconds:-0} \ ${avgdsec:-0}.${avgdmsec:-0} + # If 0 concurrency is allowed, show paused status when concurrency is 0 if ! (( concurrency )) && ! (( cron )) then if (( active )) @@ -681,6 +796,7 @@ done echo 'COMMIT;' >&3 unset count +# Final report. Calculate elapsed time and format it in human readable way. endtime=$EPOCHSECONDS (( elapsedseconds = endtime - starttime )) @@ -715,6 +831,9 @@ endtime=$EPOCHSECONDS (( cron )) || echo -en "\033[K" (( ran )) && echo +# If some tasks failed, print them. Don't print failed tasks that did not run +# to avoid confusing tasks marked as failed because they depended on ones that +# failed. if (( failed )) then echo $'\nFailed tasks:\n' @@ -801,6 +920,8 @@ then done fi +# Check if there are files that need to be renamed because their rename pattern +# changed. for destination in "${!destinationpath[@]}" do echo ' @@ -860,6 +981,8 @@ do 'vorbis') extension=ogg ;; esac (( cron )) || echo -en "$destination: rename pattern changed, renaming files...\033[K" + # Spawn perl coprocess for unicode to ascii conversion if + # needed. (( textunidecodeneeded )) && ascii echo 'BEGIN TRANSACTION;' >&3 for line in "${renamefiles[@]}" @@ -935,8 +1058,12 @@ do unset count changedcount renamefiles done +# Copy files of mime-types matching `copy_mime-type` copyFiles_action +# Remove files obsoleted by `renamme_pattern`, `ascii` or `fat32compat` changes. +# Based on `destination_files.old_filename` field, populated upon task +# completion. echo ' SELECT destination_files.id, destination_files.filename, @@ -983,6 +1110,9 @@ echo 'COMMIT;' >&3 (( cron )) || echo -en "\033[K" (( count )) && echo +# Remove empty directories in destinations. +# We blindly duplicate the source tree in the destination, so we may end up +# with empty directories. (( debug )) && echo "Purging empty directories..." for path in "${destinationpath[@]}" do