1#@!#!123s

: / bin /
Filename : pt-ioprofile
back
#!/usr/bin/env bash

# This program is part of Percona Toolkit: http://www.percona.com/software/
# See "COPYRIGHT, LICENSE, AND WARRANTY" at the end of this file for legal
# notices and disclaimers.

# ###########################################################################
# log_warn_die package
# This package is a copy without comments from the original.  The original
# with comments and its test file can be found in the Bazaar repository at,
#   lib/bash/log_warn_die.sh
#   t/lib/bash/log_warn_die.sh
# See https://launchpad.net/percona-toolkit for more information.
# ###########################################################################


set -u

PTFUNCNAME=""
PTDEBUG="${PTDEBUG:-""}"
EXIT_STATUS=0

ts() {
   TS=$(date +%F-%T | tr ':-' '_')
   echo "$TS $*"
}

info() {
   [ ${OPT_VERBOSE:-3} -ge 3 ] && ts "$*"
}

log() {
   [ ${OPT_VERBOSE:-3} -ge 2 ] && ts "$*"
}

warn() {
   [ ${OPT_VERBOSE:-3} -ge 1 ] && ts "$*" >&2
   EXIT_STATUS=1
}

die() {
   ts "$*" >&2
   EXIT_STATUS=1
   exit 1
}

_d () {
   [ "$PTDEBUG" ] && echo "# $PTFUNCNAME: $(ts "$*")" >&2
}

# ###########################################################################
# End log_warn_die package
# ###########################################################################

# ###########################################################################
# parse_options package
# This package is a copy without comments from the original.  The original
# with comments and its test file can be found in the Bazaar repository at,
#   lib/bash/parse_options.sh
#   t/lib/bash/parse_options.sh
# See https://launchpad.net/percona-toolkit for more information.
# ###########################################################################





set -u

ARGV=""           # Non-option args (probably input files)
EXT_ARGV=""       # Everything after -- (args for an external command)
HAVE_EXT_ARGV=""  # Got --, everything else is put into EXT_ARGV
OPT_ERRS=0        # How many command line option errors
OPT_VERSION=""    # If --version was specified
OPT_HELP=""       # If --help was specified
OPT_ASK_PASS=""   # If --ask-pass was specified
PO_DIR=""         # Directory with program option spec files

usage() {
   local file="$1"

   local usage="$(grep '^Usage: ' "$file")"
   echo $usage
   echo
   echo "For more information, 'man $TOOL' or 'perldoc $file'."
}

usage_or_errors() {
   local file="$1"
   local version=""

   if [ "$OPT_VERSION" ]; then
      version=$(grep '^pt-[^ ]\+ [0-9]' "$file")
      echo "$version"
      return 1
   fi

   if [ "$OPT_HELP" ]; then
      usage "$file"
      echo
      echo "Command line options:"
      echo
      perl -e '
         use strict;
         use warnings FATAL => qw(all);
         my $lcol = 20;         # Allow this much space for option names.
         my $rcol = 80 - $lcol; # The terminal is assumed to be 80 chars wide.
         my $name;
         while ( <> ) {
            my $line = $_;
            chomp $line;
            if ( $line =~ s/^long:/  --/ ) {
               $name = $line;
            }
            elsif ( $line =~ s/^desc:// ) {
               $line =~ s/ +$//mg;
               my @lines = grep { $_      }
                           $line =~ m/(.{0,$rcol})(?:\s+|\Z)/g;
               if ( length($name) >= $lcol ) {
                  print $name, "\n", (q{ } x $lcol);
               }
               else {
                  printf "%-${lcol}s", $name;
               }
               print join("\n" . (q{ } x $lcol), @lines);
               print "\n";
            }
         }
      ' "$PO_DIR"/*
      echo
      echo "Options and values after processing arguments:"
      echo
      (
         cd "$PO_DIR"
         for opt in *; do
            local varname="OPT_$(echo "$opt" | tr a-z- A-Z_)"
            eval local varvalue=\$$varname
            if ! grep -q "type:" "$PO_DIR/$opt" >/dev/null; then
               if [ "$varvalue" -a "$varvalue" = "yes" ];
                  then varvalue="TRUE"
               else
                  varvalue="FALSE"
               fi
            fi
            printf -- "  --%-30s %s" "$opt" "${varvalue:-(No value)}"
            echo
         done
      )
      return 1
   fi

   if [ $OPT_ERRS -gt 0 ]; then
      echo
      usage "$file"
      return 1
   fi

   return 0
}

option_error() {
   local err="$1"
   OPT_ERRS=$(($OPT_ERRS + 1))
   echo "$err" >&2
}

parse_options() {
   local file="$1"
   shift

   ARGV=""
   EXT_ARGV=""
   HAVE_EXT_ARGV=""
   OPT_ERRS=0
   OPT_VERSION=""
   OPT_HELP=""
   OPT_ASK_PASS=""
   PO_DIR="$PT_TMPDIR/po"

   if [ ! -d "$PO_DIR" ]; then
      mkdir "$PO_DIR"
      if [ $? -ne 0 ]; then
         echo "Cannot mkdir $PO_DIR" >&2
         exit 1
      fi
   fi

   rm -rf "$PO_DIR"/*
   if [ $? -ne 0 ]; then
      echo "Cannot rm -rf $PO_DIR/*" >&2
      exit 1
   fi

   _parse_pod "$file"  # Parse POD into program option (po) spec files
   _eval_po            # Eval po into existence with default values

   if [ $# -ge 2 ] &&  [ "$1" = "--config" ]; then
      shift  # --config
      local user_config_files="$1"
      shift  # that ^
      local IFS=","
      for user_config_file in $user_config_files; do
         _parse_config_files "$user_config_file"
      done
   else
      _parse_config_files "/etc/percona-toolkit/percona-toolkit.conf" "/etc/percona-toolkit/$TOOL.conf"
      if [ "${HOME:-}" ]; then
         _parse_config_files "$HOME/.percona-toolkit.conf" "$HOME/.$TOOL.conf"
      fi
   fi

   _parse_command_line "${@:-""}"
}

_parse_pod() {
   local file="$1"

   PO_FILE="$file" PO_DIR="$PO_DIR" perl -e '
      $/ = "";
      my $file = $ENV{PO_FILE};
      open my $fh, "<", $file or die "Cannot open $file: $!";
      while ( defined(my $para = <$fh>) ) {
         next unless $para =~ m/^=head1 OPTIONS/;
         while ( defined(my $para = <$fh>) ) {
            last if $para =~ m/^=head1/;
            chomp;
            if ( $para =~ m/^=item --(\S+)/ ) {
               my $opt  = $1;
               my $file = "$ENV{PO_DIR}/$opt";
               open my $opt_fh, ">", $file or die "Cannot open $file: $!";
               print $opt_fh "long:$opt\n";
               $para = <$fh>;
               chomp;
               if ( $para =~ m/^[a-z ]+:/ ) {
                  map {
                     chomp;
                     my ($attrib, $val) = split(/: /, $_);
                     print $opt_fh "$attrib:$val\n";
                  } split(/; /, $para);
                  $para = <$fh>;
                  chomp;
               }
               my ($desc) = $para =~ m/^([^?.]+)/;
               print $opt_fh "desc:$desc.\n";
               close $opt_fh;
            }
         }
         last;
      }
   '
}

_eval_po() {
   local IFS=":"
   for opt_spec in "$PO_DIR"/*; do
      local opt=""
      local default_val=""
      local neg=0
      local size=0
      while read key val; do
         case "$key" in
            long)
               opt=$(echo $val | sed 's/-/_/g' | tr '[:lower:]' '[:upper:]')
               ;;
            default)
               default_val="$val"
               ;;
            "short form")
               ;;
            type)
               [ "$val" = "size" ] && size=1
               ;;
            desc)
               ;;
            negatable)
               if [ "$val" = "yes" ]; then
                  neg=1
               fi
               ;;
            *)
               echo "Invalid attribute in $opt_spec: $line" >&2
               exit 1
         esac 
      done < "$opt_spec"

      if [ -z "$opt" ]; then
         echo "No long attribute in option spec $opt_spec" >&2
         exit 1
      fi

      if [ $neg -eq 1 ]; then
         if [ -z "$default_val" ] || [ "$default_val" != "yes" ]; then
            echo "Option $opt_spec is negatable but not default: yes" >&2
            exit 1
         fi
      fi

      if [ $size -eq 1 -a -n "$default_val" ]; then
         default_val=$(size_to_bytes $default_val)
      fi

      eval "OPT_${opt}"="$default_val"
   done
}

_parse_config_files() {

   for config_file in "${@:-""}"; do
      test -f "$config_file" || continue

      while read config_opt; do

         echo "$config_opt" | grep '^[ ]*[^#]' >/dev/null 2>&1 || continue

         config_opt="$(echo "$config_opt" | sed -e 's/^ *//g' -e 's/ *$//g' -e 's/[ ]*=[ ]*/=/' -e 's/[ ]+#.*$//')"

         [ "$config_opt" = "" ] && continue

         echo "$config_opt" | grep -v 'version-check' >/dev/null 2>&1 || continue

         if ! [ "$HAVE_EXT_ARGV" ]; then
            config_opt="--$config_opt"
         fi

         _parse_command_line "$config_opt"

      done < "$config_file"

      HAVE_EXT_ARGV=""  # reset for each file

   done
}

_parse_command_line() {
   local opt=""
   local val=""
   local next_opt_is_val=""
   local opt_is_ok=""
   local opt_is_negated=""
   local real_opt=""
   local required_arg=""
   local spec=""

   for opt in "${@:-""}"; do
      if [ "$opt" = "--" -o "$opt" = "----" ]; then
         HAVE_EXT_ARGV=1
         continue
      fi
      if [ "$HAVE_EXT_ARGV" ]; then
         if [ "$EXT_ARGV" ]; then
            EXT_ARGV="$EXT_ARGV $opt"
         else
            EXT_ARGV="$opt"
         fi
         continue
      fi

      if [ "$next_opt_is_val" ]; then
         next_opt_is_val=""
         if [ $# -eq 0 ] || [ $(expr "$opt" : "\-") -eq 1 ]; then
            option_error "$real_opt requires a $required_arg argument"
            continue
         fi
         val="$opt"
         opt_is_ok=1
      else
         if [ $(expr "$opt" : "\-") -eq 0 ]; then
            if [ -z "$ARGV" ]; then
               ARGV="$opt"
            else
               ARGV="$ARGV $opt"
            fi
            continue
         fi

         real_opt="$opt"

         if $(echo $opt | grep '^--no[^-]' >/dev/null); then
            local base_opt=$(echo $opt | sed 's/^--no//')
            if [ -f "$PT_TMPDIR/po/$base_opt" ]; then
               opt_is_negated=1
               opt="$base_opt"
            else
               opt_is_negated=""
               opt=$(echo $opt | sed 's/^-*//')
            fi
         else
            if $(echo $opt | grep '^--no-' >/dev/null); then
               opt_is_negated=1
               opt=$(echo $opt | sed 's/^--no-//')
            else
               opt_is_negated=""
               opt=$(echo $opt | sed 's/^-*//')
            fi
         fi

         if $(echo $opt | grep '^[a-z-][a-z-]*=' >/dev/null 2>&1); then
            val="$(echo $opt | awk -F= '{print $2}')"
            opt="$(echo $opt | awk -F= '{print $1}')"
         fi

         if [ -f "$PT_TMPDIR/po/$opt" ]; then
            spec="$PT_TMPDIR/po/$opt"
         else
            spec=$(grep "^short form:-$opt\$" "$PT_TMPDIR"/po/* | cut -d ':' -f 1)
            if [ -z "$spec"  ]; then
               option_error "Unknown option: $real_opt"
               continue
            fi
         fi

         required_arg=$(cat "$spec" | awk -F: '/^type:/{print $2}')
         if [ "$required_arg" ]; then
            if [ "$val" ]; then
               opt_is_ok=1
            else
               next_opt_is_val=1
            fi
         else
            if [ "$val" ]; then
               option_error "Option $real_opt does not take a value"
               continue
            fi 
            if [ "$opt_is_negated" ]; then
               val=""
            else
               val="yes"
            fi
            opt_is_ok=1
         fi
      fi

      if [ "$opt_is_ok" ]; then
         opt=$(cat "$spec" | grep '^long:' | cut -d':' -f2 | sed 's/-/_/g' | tr '[:lower:]' '[:upper:]')

         if grep "^type:size" "$spec" >/dev/null; then
            val=$(size_to_bytes $val)
         fi

         eval "OPT_$opt"="'$val'"

         opt=""
         val=""
         next_opt_is_val=""
         opt_is_ok=""
         opt_is_negated=""
         real_opt=""
         required_arg=""
         spec=""
      fi
   done
}

size_to_bytes() {
   local size="$1"
   echo $size | perl -ne '%f=(B=>1, K=>1_024, M=>1_048_576, G=>1_073_741_824, T=>1_099_511_627_776); m/^(\d+)([kMGT])?/i; print $1 * $f{uc($2 || "B")};'
}

# ###########################################################################
# End parse_options package
# ###########################################################################

# ###########################################################################
# tmpdir package
# This package is a copy without comments from the original.  The original
# with comments and its test file can be found in the Bazaar repository at,
#   lib/bash/tmpdir.sh
#   t/lib/bash/tmpdir.sh
# See https://launchpad.net/percona-toolkit for more information.
# ###########################################################################


set -u

PT_TMPDIR=""

mk_tmpdir() {
   local dir="${1:-""}"

   if [ -n "$dir" ]; then
      if [ ! -d "$dir" ]; then
         mkdir "$dir" || die "Cannot make tmpdir $dir"
      fi
      PT_TMPDIR="$dir"
   else
      local tool="${0##*/}"
      local pid="$$"
      PT_TMPDIR=`mktemp -d -t "${tool}.${pid}.XXXXXX"` \
         || die "Cannot make secure tmpdir"
   fi
}

rm_tmpdir() {
   if [ -n "$PT_TMPDIR" ] && [ -d "$PT_TMPDIR" ]; then
      rm -rf "$PT_TMPDIR"
   fi
   PT_TMPDIR=""
}

# ###########################################################################
# End tmpdir package
# ###########################################################################

# ###########################################################################
# alt_cmds package
# This package is a copy without comments from the original.  The original
# with comments and its test file can be found in the Bazaar repository at,
#   lib/bash/alt_cmds.sh
#   t/lib/bash/alt_cmds.sh
# See https://launchpad.net/percona-toolkit for more information.
# ###########################################################################


set -u

_seq() {
   local i="$1"
   awk "BEGIN { for(i=1; i<=$i; i++) print i; }"
}

_pidof() {
   local cmd="$1"
   if ! pidof "$cmd" 2>/dev/null; then
      ps -eo pid,ucomm | awk -v comm="$cmd" '$2 == comm { print $1 }'
   fi
}

_lsof() {
   local pid="$1"
   if ! lsof -p $pid 2>/dev/null; then
      /bin/ls -l /proc/$pid/fd 2>/dev/null
   fi
}



_which() {
   if [ -x /usr/bin/which ]; then
      /usr/bin/which "$1" 2>/dev/null | awk '{print $1}'
   elif which which 1>/dev/null 2>&1; then
      which "$1" 2>/dev/null | awk '{print $1}'
   else
      echo "$1"
   fi
}

# ###########################################################################
# End alt_cmds package
# ###########################################################################

# ###########################################################################
# Global variables
# ###########################################################################
TOOL="pt-ioprofile"

# ###########################################################################
# Subroutines
# ###########################################################################

# Read the 'lsof' and 'strace' from the file, and convert it into lines:
# pid function fd_no size timing filename
# The arguments are the files to summarize.
tabulate_strace() {
   cat > $PT_TMPDIR/tabulate_strace.awk <<EOF
   BEGIN {
      # These are function names, or partial function names, that we care about.
      # Later we will ignore any function whose name doesn't look like these.
      # Keep this in sync with wanted_pat in summarize_strace, too.
      wanted_pat  = "read|write|sync|open|close|getdents|seek|fcntl|ftrunc";
      cwd         = ""; # The process's cwd to prepend to ./ filenames later.
      mode        = 0;  # Whether we're in the lsof or strace part of the input.
   }
   /^COMMAND/ { mode = "lsof";   }
   /^Process/ { mode = "strace"; }
   {
      # Save the file descriptor and name for lookup later.
      if ( mode == "lsof" ) {
         if ( \$5 == "REG" ) {
            fd = \$4;
            gsub(/[rwu-].*/, "", fd);
            filename_for[fd] = \$9;
         }
         else if ( \$5 == "DIR" && \$4 == "cwd" ) {
            cwd = \$NF;
         }
      }
      else if ( mode == "strace" && \$1 ~ /^\[/ ) {
         pid = substr(\$2, 1, length(\$2) - 1);

         # Continuation of a previously <unfinished ...> function call
         if ( \$3 == "<..." ) {
            funcn      = \$4;
            fd         = unfinished[pid "," funcn];
            if ( fd > 0 ) {
               filename = filename_for[fd];
               if ( filename != "" ) {
                  if ( funcn ~ /open/ ) {
                     size = 0;
                  }
                  else {
                     size_field = NF - 1;
                     size       = \$size_field;
                  }
                  timing = \$NF;
                  gsub(/[<>]/, "", timing);
                  print pid, funcn, fd, size, timing, filename;
               }
            }
         }

         # The beginning of a function call (not resumed).  There are basically
         # two cases here: the whole call is on one line, and it's unfinished
         # and ends on a later line.
         else {
            funcn = substr(\$3, 1, index(\$3, "(") - 1);
            if ( funcn ~ wanted_pat ) {
               # Save the file descriptor and name for lookup later.
               if ( funcn ~ /open/ ) {
                  filename = substr(\$3, index(\$3, "(") + 2);
                  filename = substr(filename, 1, index(filename, "\\"") - 1);
                  if ( "./" == substr(filename, 1, 2) ) {
                     # Translate relative filenames into absolute ones.
                     filename = cwd substr(filename, 2);
                  }
                  fd_field         = NF - 1;
                  fd               = \$fd_field;
                  filename_for[fd] = filename;
               }

               else {
                  fd  = substr(\$3, index(\$3, "(") + 1);
                  gsub(/[^0-9].*/, "", fd);
               }

               # Save unfinished calls for later
               if ( \$NF == "...>" ) {
                  unfinished[pid "," funcn] = fd;
               }

               # Function calls that are all on one line, not <unfinished ...>
               else {
                  filename = filename_for[fd];
                  if ( filename != "" ) {
                     if ( funcn ~ /open/ ) {
                        size = 0;
                     }
                     else {
                        size_field = NF - 1;
                        size       = \$size_field;
                     }
                     timing = \$NF;
                     gsub(/[<>]/, "", timing);
                     print pid, funcn, fd, size, timing, filename;
                  }
               }
            }
         }
      }
   }
EOF
   awk -f $PT_TMPDIR/tabulate_strace.awk "$@"
}

# Takes as input the output from tabulate_strace.  Arguments are just a subset
# of the overall command-line options, but no validation is needed.  The last
# command-line option is the filename of the tabulate_strace output.
summarize_strace() {
   local func="$1"
   local cell="$2"
   local group_by="$3"
   local file="$4"

   cat > "$PT_TMPDIR/summarize_strace.awk" <<EOF
   BEGIN {
      # These are function names, or partial function names, that we care about.
      # Later we will ignore any function whose name doesn't look like these.
      # Keep this in sync with wanted_pat in tabulate_strace, too.
      wanted_pat  = "read|write|sync|open|close|getdents|seek|fcntl|ftrunc";
      wanted["1"] = "read"; # Will match pread, pread64, etc.
      wanted["2"] = "write";
      wanted["3"] = "sync";
      wanted["4"] = "open";
      wanted["5"] = "close";
      wanted["6"] = "getdents";
      wanted["7"] = "seek";
      wanted["8"] = "fcntl";
      wanted["9"] = "ftrunc";
      num_wanted  = 9;
      col_pat     = "%10d ";
      hdr_pat     = "%10s ";
      if ( "$cell" == "times" ) {
         col_pat  = "%10.6f ";
      }
   }
   {
      pid      = \$1;
      funcn    = \$2;
      fd       = \$3;
      size     = \$4;
      timing   = \$5;
      filename = \$6;
      all      = "all";
      if ( funcn ~ wanted_pat ) {
         func_names[funcn]++;
         groupby[$group_by]++;
         count[funcn "," $group_by]++;
         sizes[funcn "," $group_by] += size;
         times[funcn "," $group_by] += timing;
      }
   }
   END {
      # Choose which functions we want to print out, ordered by wanted[].
      num_functions = 0;
      if ( "$group_by" != "all" ) {
         printf(hdr_pat, "total");
      }
      for (i = 1; i <= num_wanted; i++) {
         pat = wanted[i];
         for (funcn in func_names) {
            if ( funcn ~ pat && !accepted[funcn] ) {
               num_functions++;
               funcs_to_print[num_functions] = funcn;
               accepted[funcn]++;
               if ( "$group_by" != "all" ) {
                  printf(hdr_pat, funcn);
               }
            }
         }
      }
      if ( "$group_by" != "all" ) {
         print "$group_by";
      }

      # groupby[] contains only files/pids that have been referenced by some
      # functions, so we are automatically including only files that have some
      # activity from wanted functions. We iterate through each function name
      # and print the cell of the table.
      for (thing in groupby) {
         total_count = 0;
         total_sizes = 0;
         total_times = 0;
         output      = "";
         for (i = 1; i <= num_functions; i++) {
            funcn = funcs_to_print[i];
            total_count += count[funcn "," thing];
            total_sizes += sizes[funcn "," thing];
            total_times += times[funcn "," thing];
            result       = $cell[funcn "," thing];
            if ( "$func" == "avg" ) {
               if ( count[funcn "," thing] > 0 ) {
                  result /= count[funcn "," thing];
               }
               else {
                  result = 0;
               }
            }
            if ( "$group_by" != "all" ) {
               output = output sprintf(col_pat, result);
            }
            else {
               printf(col_pat funcn "\\n", result);
            }
         }
         total_result = total_$cell;
         if ( "$func" == "avg" ) {
            if ( total_count > 0 ) {
               total_result /= total_count;
            }
            else {
               total_result = 0;
            }
         }
         printf(col_pat, total_result);
         if ( "$group_by" != "all" ) {
            print(output thing);
         }
         else {
            print "TOTAL";
         }
      }
   }
EOF

   awk -f $PT_TMPDIR/summarize_strace.awk "$file" > $PT_TMPDIR/summarized_samples
   if [ "$group_by" != "all" ]; then
      head -n1 $PT_TMPDIR/summarized_samples
      tail -n +2 $PT_TMPDIR/summarized_samples | sort -rn -k1
   else
      grep TOTAL $PT_TMPDIR/summarized_samples
      grep -v TOTAL $PT_TMPDIR/summarized_samples | sort -rn -k1
   fi
}

sigtrap() {
   warn "Caught signal, forcing exit"
   rm_tmpdir
   exit $EXIT_STATUS
}

main() {
   trap sigtrap HUP INT TERM
   if [ $# -gt 0 ]; then
      # Summarize the files the user passed in.
      tabulate_strace "$@" > $PT_TMPDIR/tabulated_samples
   else
      # There's no file to analyze, so we'll make one.
      if which strace > /dev/null 2>&1; then

         local samples=${OPT_SAVE_SAMPLES:-"$PT_TMPDIR/samples"}

         # Get the PID of the process to profile, unless the user
         # gave us it explicitly with --profile-pid.
         local proc_pid="$OPT_PROFILE_PID"
         if [ -z "$proc_pid" ]; then
            proc_pid=$(_pidof "$OPT_PROFILE_PROCESS" | awk '{print $1; exit;'})
         fi

         date

         if [ "$proc_pid" ]; then
            echo "Tracing process ID $proc_pid"

            _lsof "$proc_pid" > "$samples" 2>&1
            if [ "$?" -ne "0" ]; then
               echo "Error: could not execute lsof, error code $?"
               EXIT_STATUS=1
               return 1
            fi

            strace -T -s 0 -f -p $proc_pid >> "$samples" 2>&1 &
            if [ "$?" -ne "0" ]; then
               echo "Error: could not execute strace, error code $?"
               EXIT_STATUS=1
               return 1
            fi
            strace_pid=$!
            # sleep one second then check to make sure the strace is
            # actually running
            sleep 1
            ps -p $strace_pid > /dev/null 2>&1
            if [ "$?" -ne "0" ]; then
               echo "Cannot find strace process" >&2
               tail "$samples" >&2
               EXIT_STATUS=1
               return 1
            fi 
            # sleep for interval -1, since we did a one second sleep
            # before checking for the PID of strace
            if [ $((${OPT_RUN_TIME}-1)) -gt 0 ]; then
                sleep $((${OPT_RUN_TIME}-1))
            fi

            kill -s 2 $strace_pid

            sleep 1
            kill -s 15 $strace_pid 2>/dev/null

            # Sometimes strace leaves threads/processes in T status.
            kill -s 18 $proc_pid

            # Summarize the output we just generated.
            tabulate_strace "$samples" > $PT_TMPDIR/tabulated_samples
         else
            echo "Cannot determine PID of $OPT_PROFILE_PROCESS process" >&2
            EXIT_STATUS=1
            return 1
         fi
      else
         echo "strace is not in PATH" >&2
         EXIT_STATUS=1
         return 1
      fi
   fi

   summarize_strace  \
      $OPT_AGGREGATE \
      $OPT_CELL      \
      $OPT_GROUP_BY  \
      "$PT_TMPDIR/tabulated_samples"
}

# Execute the program if it was not included from another file.
# This makes it possible to include without executing, and thus test.
if    [ "${0##*/}" = "$TOOL" ] \
   || [ "${0##*/}" = "bash" -a "${_:-""}" = "$0" ]; then

   # Parse command line options.  We must do this first so we can
   # see if --daemonize was specified.
   mk_tmpdir
   parse_options "$0" "$@"
   usage_or_errors "$0"
   po_status=$?
   rm_tmpdir
   if [ $po_status -eq 0 ]; then
      # Make a secure tmpdir.
      mk_tmpdir
      # XXX
      # TODO: This should be quoted but because the way parse_options()
      # currently works, it flattens files in $@ (i.e. given on the cmd
      # line) into the string $ARGV.  So if we pass "$ARGV" then other
      # functions will see 1 file named "file1 file2" instead of "file1"
      # "file2".
      main $ARGV
      # Clean up.
      rm_tmpdir
   else
      [ $OPT_ERRS -gt 0 ] && EXIT_STATUS=1
   fi
   exit $EXIT_STATUS
fi

# ############################################################################
# Documentation
# ############################################################################
:<<'DOCUMENTATION'
=pod

=head1 NAME

pt-ioprofile - Watch process IO and print a table of file and I/O activity.

=head1 SYNOPSIS

Usage: pt-ioprofile [OPTIONS] [FILE]

pt-ioprofile does two things: 1) get lsof+strace for -s seconds, 2) aggregate
the result. If you specify a FILE, then step 1) is not performed.

=head1 RISKS

B<WARNING>: pt-ioprofile freezes the server and may crash the process, or
make it perform badly after detaching, or leave it in a sleeping state!
Before using this tool, please:

=over

=item * Read the tool's documentation

=item * Review the tool's known L<"BUGS">

=item * Test the tool on a non-production server

=item * Backup your production server and verify the backups

pt-ioprofile should be considered an intrusive tool, and should not be used
on production servers unless you understand and accept the risks.

=back

=head1 DESCRIPTION

pt-ioprofile uses C<strace> and C<lsof> to watch a process's IO and print out
a table of files and I/O activity.  By default, it watches the mysqld
process for 30 seconds.  The output is like:

  Tue Dec 27 15:33:57 PST 2011
  Tracing process ID 1833
       total       read      write      lseek  ftruncate filename
    0.000150   0.000029   0.000068   0.000038   0.000015 /tmp/ibBE5opS

You probably need to run this tool as root.

pt-ioprofile works by attaching C<strace> to the process using C<ptrace()>,
which will make it run very slowly until C<strace> detaches.  In addition to
freezing the server, there is some risk of the process crashing or performing
badly after C<strace> detaches from it, or of C<strace> not detaching cleanly
and leaving the process in a sleeping state.  As a result, this should be
considered an intrusive tool, and should not be used on production servers
unless you are comfortable with that.

=head1 OPTIONS

=over

=item --aggregate

short form: -a; type: string; default: sum

The aggregate function, either C<sum> or C<avg>.

If sum, then each cell will contain the sum of the values in it.
If avg, then each cell will contain the average of the values in it.

=item --cell

short form: -c; type: string; default: times

The cell contents.

Valid values are:

   VALUE  CELLS CONTAIN
   =====  =======================
   count  Count of I/O operations
   sizes  Sizes of I/O operations
   times  I/O operation timing

=item --group-by

short form: -g; type: string; default: filename

The group-by item.

Valid values are:

   VALUE     GROUPING
   =====     ======================================
   all       Summarize into a single line of output
   filename  One line of output per filename
   pid       One line of output per process ID

=item --help

Print help and exit.

=item --profile-pid

short form: -p; type: int

The PID to profile, overrides L<"--profile-process">.

=item --profile-process

short form: -b; type: string; default: mysqld

The process name to profile.

=item --run-time

type: int; default: 30

How long to profile.

=item --save-samples

type: string

Filename to save samples in; these can be used for later analysis.

=item --version

Print the tool's version and exit.

=back

=head1 ENVIRONMENT

This tool does not use any environment variables.

=head1 SYSTEM REQUIREMENTS

This tool requires the Bourne shell (F</bin/sh>).

=head1 BUGS

For a list of known bugs, see L<http://www.percona.com/bugs/pt-ioprofile>.

Please report bugs at L<https://jira.percona.com/projects/PT>.
Include the following information in your bug report:

=over

=item * Complete command-line used to run the tool

=item * Tool L<"--version">

=item * MySQL version of all servers involved

=item * Output from the tool including STDERR

=item * Input files (log/dump/config files, etc.)

=back

If possible, include debugging output by running the tool with C<PTDEBUG>;
see L<"ENVIRONMENT">.

=head1 DOWNLOADING

Visit L<http://www.percona.com/software/percona-toolkit/> to download the
latest release of Percona Toolkit.  Or, get the latest release from the
command line:

   wget percona.com/get/percona-toolkit.tar.gz

   wget percona.com/get/percona-toolkit.rpm

   wget percona.com/get/percona-toolkit.deb

You can also get individual tools from the latest release:

   wget percona.com/get/TOOL

Replace C<TOOL> with the name of any tool.

=head1 AUTHORS

Baron Schwartz

=head1 ABOUT PERCONA TOOLKIT

This tool is part of Percona Toolkit, a collection of advanced command-line
tools for MySQL developed by Percona.  Percona Toolkit was forked from two
projects in June, 2011: Maatkit and Aspersa.  Those projects were created by
Baron Schwartz and primarily developed by him and Daniel Nichter.  Visit
L<http://www.percona.com/software/> to learn about other free, open-source
software from Percona.

=head1 COPYRIGHT, LICENSE, AND WARRANTY

This program is copyright 2011-2018 Percona LLC and/or its affiliates,
2010-2011 Baron Schwartz.

THIS PROGRAM IS PROVIDED "AS IS" AND WITHOUT ANY EXPRESS OR IMPLIED
WARRANTIES, INCLUDING, WITHOUT LIMITATION, THE IMPLIED WARRANTIES OF
MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE.

This program is free software; you can redistribute it and/or modify it under
the terms of the GNU General Public License as published by the Free Software
Foundation, version 2; OR the Perl Artistic License.  On UNIX and similar
systems, you can issue `man perlgpl' or `man perlartistic' to read these
licenses.

You should have received a copy of the GNU General Public License along with
this program; if not, write to the Free Software Foundation, Inc., 59 Temple
Place, Suite 330, Boston, MA  02111-1307  USA.

=head1 VERSION

pt-ioprofile 3.1.0

=cut

DOCUMENTATION