ziggy.hdmc.checkpoint_frame_save

Package ziggy :: Package hdmc :: Module checkpoint_frame_save |
[hide private]
[frames] | no frames]

# Source Code for Module ziggy.hdmc.checkpoint_frame_save

     1  #! /usr/bin/env python
     2
     3  from glob import glob
     4  import sys, os, subprocess, shlex, random, time, re
     5
     6 -def main():
     7          wait_counter = 1
     8          time.sleep(random.random())
     9          all_checkpoints = ['0', '1', '2', '3', '4', '5', '6']
    10          this_checkpoint = random.choice(all_checkpoints)
    11          this_checkpoint_start = this_checkpoint+'_start'
    12          this_checkpoint_end = this_checkpoint+'_end'
    13          current_checkpoints = glob('/usr/local/hadoop/user-tmp/dwmclary/hdmc_checkpoints/*')
    14          final_checkpoints = glob('/usr/local/hadoop/user-tmp/dwmclary/hdmc_checkpoints/*_end')
    15          while len(final_checkpoints) < len(all_checkpoints):
    16                  for i in range(len(current_checkpoints)):
    17                          current_checkpoints[i] = re.sub('/usr/local/hadoop/user-tmp/dwmclary/hdmc_checkpoints/', '', current_checkpoints[i])
    18                  while this_checkpoint_end in current_checkpoints:
    19                          this_checkpoint = random.choice(all_checkpoints)
    20                          this_checkpoint_start = this_checkpoint+'_start'
    21                          this_checkpoint_end = this_checkpoint+'_end'
    22                          current_checkpoints = glob('/usr/local/hadoop/user-tmp/dwmclary/hdmc_checkpoints/*')
    23                          final_checkpoints = glob('/usr/local/hadoop/user-tmp/dwmclary/hdmc_checkpoints/*_end')
    24                          for i in range(len(current_checkpoints)):
    25                                  current_checkpoints[i] = re.sub('/usr/local/hadoop/user-tmp/dwmclary/hdmc_checkpoints/', '', current_checkpoints[i])
    26                          if len(final_checkpoints) == len(all_checkpoints):
    27                                  exit()
    28
    29                  subprocess.call(['touch','/usr/local/hadoop/user-tmp/dwmclary/hdmc_checkpoints'+'/'+this_checkpoint+'_start'])
    30                  subprocess.call(['chmod','777','/usr/local/hadoop/user-tmp/dwmclary/hdmc_checkpoints'+'/'+this_checkpoint+'_start'])
    31                  os.system('chmod a+rwx fetch_books.py')
    32                  argf = open('./'+this_checkpoint).readlines()
    33                  for i in range(len(argf)):
    34                          argf[i] = argf[i].strip()
    35                  cmd = ['./fetch_books.py']+argf
    36                  p = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
    37                  output, error = p.communicate()
    38                  sts = p.wait()
    39                  output = output.strip()
    40                  if len(output) > 0:
    41                          line_count = 0
    42                          newline = '\n'
    43                          for line in output.split(newline):
    44                                  print this_checkpoint+'==HDMC_CHECKPOINT==LINE=='+ str(line_count) + '=='+line.strip()
    45                                  line_count += 1
    46                  if len(error.strip()) > 0:
    47                          print >> sys.stderr, error.strip()
    48                          os.system('rm /usr/local/hadoop/user-tmp/dwmclary/hdmc_checkpoints'+'/'+this_checkpoint)
    49                          exit(1)
    50                  else:
    51                          subprocess.call(['touch','/usr/local/hadoop/user-tmp/dwmclary/hdmc_checkpoints'+'/'+this_checkpoint+'_end'])
    52                          subprocess.call(['chmod','777','/usr/local/hadoop/user-tmp/dwmclary/hdmc_checkpoints'+'/'+this_checkpoint+'_end'])
    53                  os.system('rm /usr/local/hadoop/user-tmp/dwmclary/hdmc_checkpoints/'+this_checkpoint+'_start')
    54                  subprocess.call(['touch','/usr/local/hadoop/user-tmp/dwmclary/hdmc_checkpoints'+'/'+this_checkpoint+'_end'])
    55                  this_checkpoint = random.choice(all_checkpoints)
    56                  current_checkpoints = glob('/usr/local/hadoop/user-tmp/dwmclary/hdmc_checkpoints/*')
    57
    58
    59  if __name__ == "__main__":
    60          main()
    61