SHELL TIPS

Alternate Link

Array Operations

#Append
var=( ${array[@]-} $(echo "$variable") )

#Get Unique elements from any array
echo ${oldArray[@]}|awk '{for(i=1;i<NF;i++) printf "%sn",$i}' | sort -u

#Check if element is present in an array

File Operations

#list files greater than 10bytes
find *.csv.gz -type f -size +10

#list files less than 11 bytes
find *.csv.gz -type f -size -11

#add line number
cat -n infile > outfile

#fetch nth line. Example fetch 74th line from file
awk 'NR==74' file
head -n 74 file | tail -1

#fetch nth line from all the files matching certain criteria
find path_to_folder -n "file_pattern_* | xargs -n 1 awk 'NR==74'

#add two files column wise using tab as delimiter
paste -d$'t' file1 file2 > newfile

# remove ctrl-M from end of line. Ctrl-M is dos format to indicate end of line. Use tr command to remove it
cat file | tr -d '15' > newfile

#change delimiter from tab to pipe
cat file | tr "t" '|' > newfile

# Remove lines present in file 1 and file 2
cat file1 file2 | sort | uniq -u > newfile

#Read file line by line
while read line
do
  echo $line
done < input_file

#Get number of lines and columns in a file (assume tab delimited file)
lines=`awk ' END { print NR } ' filename.tab`
cols=`awk 'BEGIN {FS="\t"}; END{print NF}' filename.tab`

#Random selection (get 1% of data. Change .01 if you need more)
cat file1 | awk 'BEGIN {srand()} !/^$/ { if (rand() <= .01) print $0}' > file2

Postgres Command Line Utilities

#Delete all rows from all tables. In xargs command -n1 indicates that process each line separately and -I table refers to the table.
psql -U user database -c "select relname, n_live_tup from pg_stat_user_tables order by n_live_tup desc;" | awk -F'|' '{if($2 > 0) print $1}' | xargs -n1 -I table psql -U user database -c "delete from table;"

#display row count for all tables in a database
psql -U user database -c "select tablename from pg_tables where tablename not like 'pg_%' and tablename not like 'sql_%';" | xargs -n1 -I table psql -U user database -c "select 'table' as table, count(*) from table;"

About these ads