Investigate splitting big files about giftstick HOT 3 CLOSED

import mmap                                                                      
                                                                                   
from google.cloud import storage                                                 
                                                                                 
def slice_big(file, slices, total_size, filename):                                                     
  storage_client = storage.Client()                                              
  bucket = storage_client.bucket('somebbucket')                                  
                                                                                   
  if int(slices) < 1:                                                            
    raise Exception('Need at least 1 slice')                                     
                                                                                 
  number_of_pages = int(total_size / int(slices) /  mmap.PAGESIZE)               
                                                                                 
  slice_size = number_of_pages * mmap.PAGESIZE                                   
                                                                                 
  for i, seek in enumerate(range(0, total_size, slice_size)):                    
    length = slice_size                                                          
    if seek+slice_size > total_size: 
      # No trying to read past end of filedescriptor                                          
      length = total_size - seek                                                 
    print(f'sending slice {i} from bytes {seek} to {seek+length}')               
    m = mmap.mmap(                                                               
        fd.fileno(), length=length, offset=seek, access=mmap.ACCESS_READ)        
    blob = bucket.blob('large_split/'+filename+f'_{i}')                          
    blob.upload_from_file(m)                                               
                                                                                   
 slice_big(open('/dev/loop0', 'rb'), 12, 1048576000, 'big')   

import mmap                                                                      
                                                                                   
from google.cloud import storage                                                                                                                                
                                                                                   
def slice_big(fd, slices, total_size, filename, skip_exist=False):               
  storage_client = storage.Client()                  
  bucket = storage_client.bucket('somebbucket')                              
                                                                                   
  if int(slices) < 1:                                                            
    raise Exception('Need at least 1 slice')                                     
  # mmap requires that the an offset is a multiple of mmap.PAGESIZE              
  # so we can't just equally divide the total size in the specified number of    
  # slices.                                                                      
  number_of_pages = int(total_size / int(slices) /  mmap.PAGESIZE)               
                                                                                 
  slice_size = number_of_pages * mmap.PAGESIZE                                   
                                                                                 
  for i, seek in enumerate(range(0, total_size, slice_size)):                    
    blob_path = 'large_split/'+filename+f'_{i}'                                  
    blob = bucket.blob(blob_path)                                                
                                                                                 
    if skip_exist and blob.exists():                                             
      print(f'{blob_path} already exists, skipping')                             
      continue                                                                   
                                                                                 
    length = slice_size                                                          
    if seek+slice_size > total_size:                                             
      length = total_size - seek                                                 
    print(f'sending slice {i} from bytes {seek} to {seek+length}')               
    m = mmap.mmap(                                                               
        fd.fileno(), length=length, offset=seek, access=mmap.ACCESS_READ)        
    blob.upload_from_file(m)                                                     
                                                                                 
slice_big(open('/dev/loop0', 'rb'), 12, 1048576000, 'biggy', skip_exist=True)