ziggy.hdmc.hdfs.hdfs_access

Package ziggy :: Package hdmc :: Package hdfs :: Module hdfs_access |
[hide private]
[frames] | no frames]

# Source Code for Module ziggy.hdmc.hdfs.hdfs_access

     1  '''
     2  Created on May 17, 2010
     3
     4  @author: dwmclary
     5  '''
     6
     7  from . import hdfs_config as config
     8  import subprocess
     9
    10
    11 -def run(args):
    12      '''Generic runner for HDFS access'''
    13      p = subprocess.Popen(args, stdout=subprocess.PIPE, stderr = subprocess.PIPE)
    14      r = p.communicate()
    15      result = {"stdout":r[0], "stderr":r[1]}
    16      return result
    17
    18
    19 -def cat(filename):
    20      '''Runs HDFS cat on a filename or glob passed as a string
    21      Returns the result as a string read from STOUT'''
    22      args = [config.hadoop, "dfs", "-cat", filename]
    23      return run(args)
    24
    25
    26 -def ls(filename):
    27      '''Runs HDFS ls on a directory or glob passed as a string
    28      Returns the result as a string read from STOUT'''
    29      args = [config.hadoop, "dfs", "-ls", filename]
    30      return run(args)
    31
    32 -def tail(filename):
    33      args = [config.hadoop, "dfs", "-tail", filename]
    34      return run(args)
    35
    36 -def mv(filename1, filename2):
    37      args = [config.hadoop, "dfs", "-mv", filename1, filename2]
    38      return run(args)
    39
    40 -def rm(filename):
    41      '''Runs HDFS rmr on a filename or glob passed as a string
    42      Returns the result as a string read from STOUT'''
    43      args = [config.hadoop, "dfs", "-rmr", filename]
    44      return run(args)
    45
    46 -def delete(filename):
    47      '''Runs HDFS rmr on a filename or glob passed as a string
    48      Returns the result as a string read from STOUT'''
    49      args = [config.hadoop, "dfs", "-rm", filename]
    50      return run(args)
    51
    52 -def mkdir(dirname):
    53      '''Runs HDFS mkdir on a directory name as a string
    54      Returns the result as a string read from STOUT'''
    55      args = [config.hadoop, "dfs", "-mkdir", dirname]
    56      return run(args)
    57
    58 -def get_merge(pattern, dst):
    59      '''Runs HDFS rmr on a filename or glob passed as a string
    60      Returns the result as a string read from STOUT'''
    61      args = [config.hadoop, "dfs", "-getmerge", pattern, dst]
    62      return run(args)
    63
    64 -def mv(src, dst):
    65      '''Runs HDFS mv: move src HDFS file to dst
    66      Returns the result as a string read from STOUT'''
    67      args = [config.hadoop, "dfs", "-mv", src, dst]
    68      return run(args)
    69
    70
    71
    72 -def copyToHDFS(filename, hdfs_filename):
    73      args = [config.hadoop, "dfs", "-put", filename, hdfs_filename]
    74      return run(args)
    75
    76 -def copyFromHDFS(hdfs_filename, filename):
    77      args = [config.hadoop, "dfs", "-get", hdfs_filename, filename]
    78      return run(args)
    79