diff options
| -rw-r--r-- | .gitignore | 3 | ||||
| -rw-r--r-- | readme.rst | 138 | ||||
| -rwxr-xr-x | teaqueue-client | 47 | ||||
| -rwxr-xr-x | teaqueue-server | 53 | ||||
| -rwxr-xr-x | worker_examples/echo_sleep.sh | 5 | ||||
| -rw-r--r-- | worker_examples/remote_transcode.sh | 6 |
6 files changed, 248 insertions, 4 deletions
diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..98edcf1 --- /dev/null +++ b/.gitignore @@ -0,0 +1,3 @@ +/queue.txt +/queue.txt.old +/done.txt @@ -1,9 +1,121 @@ Simple Text Queue -================= +################# + +Why and what it is +================== + +**teaqueue** is a command pair able to process sequentially items such as +files from a list of filenames in a text file by calling a given command for +each of them. You use **GNU** tools like `find` to populate your queue, or use +**vim** to reorder it. + +It can be used to: + +- transcode a batch of video files such as your preferred technical + presentations to a format accepted by your mobile phone, + +- retrieve a bunch of URLs or other remote operation, + +- sequential operation on anything that can be encoded on a single line (json + data, base64 data, etc). + +It is really useful for operations taking a lot of time when you want to be +able to stop processing items in the queue and resume it. Thus, when stopped +you can optionally change the order of items, add items, or remove items from +the queue. Why `tea` ??? +------------- + +Because I love tea. + +Usage +===== + +Usage is linked directly to manipulation of simple text files. You use text +files to make the queue, to add or remove items from it, to change priority of +items. *UNIX is the way*. + +I give some examples in this section. Take into account `teaqueue-server` +takes `queue.txt` in the current directory by default as queue file modifying +it when interrupted to remove processed items from it. By default, processed +items goes to `done.txt`. This files serves to remove processed items from +`queue.txt` when the queue is interrupted and serves as a log file for +processed items. + +Video batch transcoding +----------------------- + +Firstly list filenames in the queue and serve the queue:: + + find -type f -iname '*.webm' > queue.txt + teaqueue-server + +Then ask `teaqueue-client` to process each files with a transcoding command:: + + teaqueue-client transcode_my_file.sh + +This is all if you just want to process each file and do not interrupt it. If +you want to interrupt processing of the queue, modify it and resume it, first +press `Ctrl-C` in the `teaqueue-server` console and modify `queue.txt` +accordingly:: + + find ../another_dir -type '*.webm' >> queue.txt + +Or reorder some items:: + + vim queue.txt + +Then run again `teaqueue-server`, clients will continue to ask new items from +the queue:: -- Because I love tea. + teaqueue-server transcode_my_file.sh + +`transcode_my_file.sh` is a called a worker and is a simple script which calls +`ffmpeg` or another program to convert your files. You can find examples of +workers inside the `worker_examples` directory. + +Write a worker +============== + +In order to be useful you certainly have to write a worker script. Writing +a worker is simple, all you have to do is to store the standard input line and +work with it. + +Echo worker +----------- + +For example, a useless worker, but simple to understand the principle is just +a worker echo each item it receive: + +.. code:: shell + #!/bin/sh + line="$(cat)" + echo "Printing line '$line' and sleeping $1 seconds" + sleep $1 + +You can use it with a `teaqueue-client`:: + + teaqueue-client ./echo_sleep.sh 10 + +Remote files +------------ + +If you want to be able to use `teaqueue-client` on another host than the +`teaqueue-server` you can download the files (sharing it on the server side is +done through anything you want) from the host, then work on it and push back +the transformed files. Just write the worker in consequence: + +.. code:: shell + + #!/bin/sh + filename="$(cat)" + echo "$filename" | nc -q0 serverin 1338 > file + transcode.sh file > fileout + (echo "$filename"; cat fileout) | nc -q0 serverout 1338 + +This example assume you share your files with simple netcat commands, but you +can adapt with anything you want: wget, scp, etc. History ======= @@ -17,5 +129,23 @@ processed, you could write something like this:: comm -23 queue.txt done.txt > remaining_queue.txt But if this is something you want to do again and again, it might be -convenient to write a little wrapper to do it for us. This is how teaqueue is -born. +convenient to write a little wrapper to do it for us. This is how **teaqueue** +was born. + +Limitations +=========== + +**teaqueue** is not perfect and does not try to be. There is no security and +some rare race conditions. + +Known race condition: + +- If someone interrupts `teaqueue-server` just after the line has been sent + but before it gets printed out, the line will not be filtered in the + `queue.txt` file. Thus it will be send again to the next demanding client. + +License +======= + +**teaqueue** is shared to you under the GPL-3 license conditions. See +`LICENSE.txt`. diff --git a/teaqueue-client b/teaqueue-client new file mode 100755 index 0000000..c52d01e --- /dev/null +++ b/teaqueue-client @@ -0,0 +1,47 @@ +#!/bin/sh + +set -eu + +# default options +host="127.0.0.1" +port=1340 +retry_timeout=10 + +usage() +{ + cat <<EOF +Usage: $0 [OPTIONS...] command [args...] + +Connects to a teaqueue-server and call "command" with a line in stdin. + +Options: + -h, --help show usage + -H, --host what teaqueue-server host (default: 127.0.0.1) + -p, --port what port to use (default: 1340) +EOF +} + +# arg parsing +TEMP=$(getopt -o hH:p: --long help,host,port -n teaqueue-client -- "$@") +if [ $? != 0 ]; then echo "Terminating..." >&2; exit 1; fi +eval set -- "$TEMP" +while true; do + case "$1" in + -h|--help) usage; exit 0;; + -H|--host) host="$2"; shift 2;; + -p|--port) port="$2"; shift 2;; + --) shift; break;; + *) echo "Arguments parsing error" >&2; exit 1;; + esac +done + +if test $# -lt 1; then + echo "You must put a command to call" >&2; exit 1 +fi + +set +e + +while true; do + while ! nc -q0 "$host" "$port"; do sleep $retry_timeout; done | "$@" + sleep 1 +done diff --git a/teaqueue-server b/teaqueue-server new file mode 100755 index 0000000..02c8b6a --- /dev/null +++ b/teaqueue-server @@ -0,0 +1,53 @@ +#!/bin/bash +# use bash for trap + +set -eu + +# default options +queuefn="queue.txt" +donefn="done.txt" +port=1340 + +usage() +{ + cat <<EOF +Usage: $0 [OPTIONS...] + +Feeds lines from a text file to teaqueue-clients. + +Options: + -h, --help show usage + -p, --port what port to use (default: 1340) + -q, --queue what filename to use as input queue + -d, --done what filename to use for storing done items +EOF +} + +# arg parsing +TEMP=$(getopt -o hH:p: --long help,host,port -n teaqueue-client -- "$@") +if [ $? != 0 ]; then echo "Terminating..." >&2; exit 1; fi +eval set -- "$TEMP" +while true; do + case "$1" in + -h|--help) usage; exit 0;; + -p|--port) port="$2"; shift 2;; + -q|--queue) queuefn="$2"; shift 2;; + -d|--done) donefn="$2"; shift 2;; + --) shift; break;; + *) echo "Arguments parsing error" >&2; exit 1;; + esac +done + +my_exit() +{ + mv "${queuefn}" "${queuefn}.old" + comm --nocheck-order -23 "${queuefn}.old" "${donefn}" > "${queuefn}" + exit 0 +} + +trap my_exit SIGINT SIGTERM + +cat "${queuefn}" | while read -r line; do + printf "%s\n" "$line" | nc -q0 -l -p $port + printf "%s\n" "$line" +done > "${donefn}" diff --git a/worker_examples/echo_sleep.sh b/worker_examples/echo_sleep.sh new file mode 100755 index 0000000..7f76c0e --- /dev/null +++ b/worker_examples/echo_sleep.sh @@ -0,0 +1,5 @@ +#!/bin/sh + +line="$(cat)" +echo "Printing line '$line' and sleeping $1 seconds" +sleep $1 diff --git a/worker_examples/remote_transcode.sh b/worker_examples/remote_transcode.sh new file mode 100644 index 0000000..f408731 --- /dev/null +++ b/worker_examples/remote_transcode.sh @@ -0,0 +1,6 @@ +#!/bin/sh +filename="$(cat)" +echo "Downloading file..." +scp remote:"$filename" . +ffmpeg $(basename $filename) -o out/$(basename $filename) +scp out/$(basename $filename) remote:out/ |