with open('data.txt', 'r') as f:
    data = f.read()

• Directory Listing in Legacy Python Versions
• Directory Listing in Modern Python Versions
• Listing All Files in a Directory
• Listing Subdirectories

built-in os module has a number of useful functions that can be used to list directory contents and filter the results
to get a list of all the files and folders in a particular directory in the filesystem

• use os.listdir() in legacy versions of Python
• os.scandir() in Python 3

>>> import os
>>> entries = os.listdir('my_directory/')
>>> os.listdir('my_directory/')
['sub_dir_c', 'file1.py', 'sub_dir_b', 'file3.txt', 'file2.csv', 'sub_dir']

>>> entries = os.listdir('my_directory/')
>>> for entry in entries:
...     print(entry)
...
...
sub_dir_c
file1.py
sub_dir_b
file3.txt
file2.csv
sub_dir

>>> import os
>>> entries = os.scandir('my_directory/')
>>> entries
<posix.ScandirIterator object at 0x7f5b047f3690>

import os

with os.scandir('my_directory/') as entries:
    for entry in entries:
        print(entry.name)

another way to get a directory listing is to use the pathlib module

from pathlib import Path

entries = Path('my_directory/')
for entry in entries.iterdir():
    print(entry.name)

pathlib offers a set of classes featuring most of the common operations on paths in an easy, object-oriented way
using pathlib is more if not equally efficient as using the functions in os
benefit of using pathlib over os is that it reduces the number of imports needed to make to manipulate filesystem paths
for further info on Python's pathlib Module

directory-listing functions

import os

# List all files in a directory using os.listdir
basepath = 'my_directory/'
for entry in os.listdir(basepath):
    # only print out filenames and not directories
    if os.path.isfile(os.path.join(basepath, entry)):
        print(entry)

import os

# List all files in a directory using scandir()
basepath = 'my_directory/'
with os.scandir(basepath) as entries:
    for entry in entries:
        if entry.is_file():
            print(entry.name)

• os.listdir()
• os.Path()

import os

# List all subdirectories using os.listdir
basepath = 'my_directory/'
for entry in os.listdir(basepath):
    if os.path.isdir(os.path.join(basepath, entry)):
        print(entry)

from pathlib import Path

# List all subdirectory using pathlib
basepath = Path('my_directory/')
for entry in basepath.iterdir():
    if entry.is_dir():
        print(entry.name)

>>> import os
>>> with os.scandir('my_directory/') as dir_contents:
...     for entry in dir_contents:
...         info = entry.stat()
            # list time file was last modified
...         print(info.st_mtime)
...
1539032199.0052035
1539032469.6324475
1538998552.2402923
1540233322.4009316
1537192240.0497339
1540266380.3434134

the pathlib module has corresponding methods for retrieving file information that give the same results

>>> from pathlib import Path
>>> current_dir = Path('my_directory')
>>> for path in current_dir.iterdir():
...     info = path.stat()
        # list time file was last modified
...     print(info.st_mtime)
...
1539032199.0052035
1539032469.6324475
1538998552.2402923
1540233322.4009316
1537192240.0497339
1540266380.3434134

from datetime import datetime
from os import scandir

def convert_date(timestamp):
    d = datetime.utcfromtimestamp(timestamp)
    formated_date = d.strftime('%d %b %Y')
    return formated_date

def get_files():
    dir_entries = scandir('my_directory/')
    for entry in dir_entries:
        if entry.is_file():
            info = entry.stat()
            print(f'{entry.name}\t Last Modified: {convert_date(info.st_mtime)}')

• %d - the day of the month
• %b - the month, in abbreviated form
• %Y - the year

• Creating a Single Directory
• Creating Multiple Directories

to create a single directory, pass a path to the directory as a parameter to os.mkdir()

import os

os.mkdir('example_directory/')

from pathlib import Path

p = Path('example_directory/')
p.mkdir()

from pathlib import Path

p = Path('example_directory')
try:
    p.mkdir()
except FileExistsError as exc:
    print(exc)

from pathlib import Path

p = Path('example_directory')
p.mkdir(exist_ok=True)

to create a group of directories like 2018/10/05

import os

os.makedirs('2018/10/05')

.
|
└── 2018/
    └── 10/
        └── 05/

import os

os.makedirs('2018/10/05', mode=0o770)

octal values are used to represent file permissions and are specified with a leading 0o or 0O
each digit in the octal number represents a specific set of permissions for the owner, group, and others, respectively
the digits range from 0 to 7, where each bit represents a permission: read (4), write (2), and execute (1). 0o755 represents rwxr-xr-x permissions

• the owner has read, write, and execute permissions (7)
• the group has read and execute permissions (5)
• others have read and execute permissions (5)

• Using String Methods
• Simple Filename Pattern Matching Using fnmatch
• More Advanced Pattern Matching
• Filename Pattern Matching Using glob

methods and functions available for pattern matching

• endswith() and startswith() string methods
• fnmatch.fnmatch()
• glob.glob()
• pathlib.Path.glob()

.
├── sub_dir/
|   ├── file1.py
|   └── file2.py
|
├── admin.py
├── data_01_backup.txt
├── data_01.txt
├── data_02_backup.txt
├── data_02.txt
├── data_03_backup.txt
├── data_03.txt
└── tests.py

>>> import os

>>> # Get .txt files
>>> for f_name in os.listdir('some_directory'):
...     if f_name.endswith('.txt'):
...         print(f_name)

>>> import os
>>> import fnmatch

>>> for file_name in os.listdir('some_directory/'):
...     if fnmatch.fnmatch(file_name, '*.txt'):
...         print(file_name)

• the word 'data'
• a number between a set of underscores
• the word 'backup'

>>> for filename in os.listdir('.'):
...     if fnmatch.fnmatch(filename, 'data_*_backup.txt'):
...         print(filename)

how to use glob to search for all Python (.py) source files in the current directory

>>> import glob
>>> glob.glob('*.py')
['admin.py', 'tests.py']

>>> import glob
>>> for file in glob.iglob('**/*.py', recursive=True):
...     print(file)

admin.py
tests.py
sub_dir/file1.py
sub_dir/file2.py

>>> from pathlib import Path
>>> p = Path('.')
>>> for name in p.glob('*.p*'):
...     print(name)

admin.py
scraper.py
docs.pdf

.
|
├── folder_1/
|   ├── file1.py
|   ├── file2.py
|   └── file3.py
|
├── folder_2/
|   ├── file4.py
|   ├── file5.py
|   └── file6.py
|
├── test1.txt
└── test2.txt

# Walking a directory tree and printing the names of the directories and files
for dirpath, dirnames, files in os.walk('.'):
    print(f'Found directory: {dirpath}')
    for file_name in files:
        print(file_name)

1. name of the current folder
2. list of folders in the current folder
3. list of files in the current folder

Found directory: .
test1.txt
test2.txt
Found directory: ./folder_1
file1.py
file3.py
file2.py
Found directory: ./folder_2
file4.py
file5.py
file6.py

for dirpath, dirnames, files in os.walk('.', topdown=False):
    print(f'Found directory: {dirpath}')
    for file_name in files:
        print(file_name)

from tempfile import TemporaryFile

# Create a temporary file and write some data to it
fp = TemporaryFile('w+t')
fp.write('Hello universe!')

# Go back to the beginning and read data from file
fp.seek(0)
data = fp.read()

# Close the file, after which it will be removed
fp.close()

temporary files and directories are stored in a special system directory for storing temporary files
Python searches a standard list of directories to find one that the user can create files in
On Windows the directories are C:\TEMP, C:\TMP, \TEMP, and \TMP, in that order

.TemporaryFile() is also a context manager so it can be used in conjunction with the with statement

with TemporaryFile('w+t') as fp:
    fp.write('Hello universe!')
    fp.seek(0)
    fp.read()
# File is now closed and removed

>>> import tempfile
>>> with tempfile.TemporaryDirectory() as tmpdir:
...     print('Created temporary directory ', tmpdir)
...     os.path.exists(tmpdir)
...
Created temporary directory  /tmp/tmpoxbkrm6c
True

>>> # Directory contents have been removed
...
>>> tmpdir
'/tmp/tmpoxbkrm6c'
>>> os.path.exists(tmpdir)
False

• Deleting Files in Python
• Deleting Directories
• Deleting Entire Directory Trees

• pathlib.Path.unlink()
• os.remove()
• os.unlink()

import os

data_file = 'C:\\Users\\vuyisile\\Desktop\\Test\\data_0.txt'
os.remove(data_file)

data_file = 'C:\\Users\\vuyisile\\Desktop\\Test\\data_1.txt'
os.unlink(data_file)

import os

data_file = 'home/data.txt'

# If the file exists, delete it
if os.path.isfile(data_file):
    os.remove(data_file)
else:
    print(f'Error: {data_file} not a valid filename')

using exception handling when deleting fiels

import os

data_file = 'home/data.txt'

# Use exception handling
try:
    os.remove(data_file)
except OSError as e:
    print(f'Error: {data_file} : {e.strerror}')

from pathlib import Path

data_file = Path('home/data.txt')

try:
    data_file.unlink()
except IsADirectoryError as e:
    print(f'Error: {data_file} : {e.strerror}')

a Python program has the same permissions as the user running it
if the user does not have permission to delete the file, a PermissionError is raised

• os.rmdir()
• pathlib.Path.rmdir()
• shutil.rmtree()

import os

trash_dir = 'my_documents/bad_dir'

try:
    os.rmdir(trash_dir)
except OSError as e:
    print(f'Error: {trash_dir} : {e.strerror}')

from pathlib import Path

trash_dir = Path('my_documents/bad_dir')

try:
    trash_dir.rmdir()
except OSError as e:
    print(f'Error: {trash_dir} : {e.strerror}')

import shutil

trash_dir = 'my_documents/bad_dir'

try:
    shutil.rmtree(trash_dir)
except OSError as e:
    print(f'Error: {trash_dir} : {e.strerror}')

import os

for dirpath, dirnames, files in os.walk('.', topdown=False):
    try:
        os.rmdir(dirpath)
    except OSError as ex:
        pass

• Copying Files in Python
• Copying Directories
• Moving Files and Directories
• Renaming Files and Directories

Python ships with the shutil module
shutil is short for shell utilities
provides a number of high-level operations on files to support copying, archiving, and removal of files and directories

import shutil

src = 'path/to/file.txt'
dst = 'path/to/dest_dir'
shutil.copy(src, dst)

import shutil

src = 'path/to/file.txt'
dst = 'path/to/dest_dir'
shutil.copy2(src, dst)

• a source directory
• the destination directory

>>> import shutil
>>> shutil.copytree('data_1', 'data1_backup')
'data1_backup'

>>> import shutil
>>> shutil.move('dir_1/', 'backup/')
'backup'

>>> os.rename('first.zip', 'first_01.zip')

>>> from pathlib import Path
>>> data_file = Path('data_01.txt')
>>> data_file.rename('data.txt')

• Reading ZIP Files
• Extracting ZIP Archives
• Extracting Data From Password Protected Archives
• Creating New ZIP Archives
• Opening TAR Archives
• Extracting Files From a TAR Archive
• Creating New TAR Archives
• Working With Compressed Archives

the two most common archive types are ZIP and TAR
Python programs can create, read, and extract data from archives

import zipfile

with zipfile.ZipFile('data.zip', 'r') as zipobj:
    pass

.
|
├── sub_dir/
|   ├── bar.py
|   └── foo.py
|
├── file1.py
├── file2.py
└── file3.py

import zipfile

with zipfile.ZipFile('data.zip', 'r') as zipobj:
    zipobj.namelist()

['file1.py', 'file2.py', 'file3.py', 'sub_dir/', 'sub_dir/bar.py', 'sub_dir/foo.py']

import zipfile

with zipfile.ZipFile('data.zip', 'r') as zipobj:
    bar_info = zipobj.getinfo('sub_dir/bar.py')
    bar_info.file_size

import zipfile

with zipfile.ZipFile('data.zip', 'r') as zipobj:
    bar_info = zipobj.getinfo('sub_dir/bar.py')
    bar_info.file_size

15277

• the date the files were last modified
• their compressed sizes
• their full filenames

>>> import zipfile
>>> import os

>>> os.listdir('.')
['data.zip']

>>> data_zip = zipfile.ZipFile('data.zip', 'r')

>>> # Extract a single file to current directory
>>> data_zip.extract('file1.py')
'/home/terra/test/dir1/zip_extract/file1.py'

>>> os.listdir('.')
['file1.py', 'data.zip']

>>> # Extract all files into a different directory
>>> data_zip.extractall(path='extract_dir/')

>>> os.listdir('.')
['file1.py', 'extract_dir', 'data.zip']

>>> os.listdir('extract_dir')
['file1.py', 'file3.py', 'file2.py', 'sub_dir']

>>> data_zip.close()

>>> import zipfile

>>> with zipfile.ZipFile('secret.zip', 'r') as pwd_zip:
...     # Extract from a password protected archive
...     pwd_zip.extractall(path='extract_dir', pwd='Quish3@o')

>>> import zipfile

>>> file_list = ['file1.py', 'sub_dir/', 'sub_dir/bar.py', 'sub_dir/foo.py']
>>> with zipfile.ZipFile('new.zip', 'w') as new_zip:
...     for name in file_list:
...         new_zip.write(name)

>>> # Open a ZipFile object in append mode
>>> with zipfile.ZipFile('new.zip', 'a') as new_zip:
...     new_zip.write('data.txt')
...     new_zip.write('latin.txt')

to read from an archive

import tarfile

with tarfile.open('example.tar', 'r') as tar_file:
    print(tar_file.getnames())

use the 'r', 'w' or 'a' modes to open an uncompressed TAR file for reading, writing, and appending, respectively
to open compressed TAR files, pass in a mode argument to tarfile.open() in the form filemode[:compression]
possible modes TAR files can be opened in:

.open() defaults to 'r' mode
to read an uncompressed TAR file and retrieve the names of the files in it use .getnames()

>>> import tarfile

>>> tar = tarfile.open('example.tar', mode='r')
>>> tar.getnames()
['CONTRIBUTING.rst', 'README.md', 'app.py']

>>> for entry in tar.getmembers():
...     print(entry.name)
...     print(' Modified:', time.ctime(entry.mtime))
...     print(' Size    :', entry.size, 'bytes')
...     print()
CONTRIBUTING.rst
 Modified: Sat Nov  1 09:09:51 2018
 Size    : 402 bytes

README.md
 Modified: Sat Nov  3 07:29:40 2018
 Size    : 5426 bytes

app.py
 Modified: Sat Nov  3 07:29:13 2018
 Size    : 6218 bytes

• .extract()
• .extractfile()
• .extractall()

>>> tar.extract('README.md')
>>> os.listdir('.')
['README.md', 'example.tar']

>>> tar.extractall(path="extracted/")

>>> f = tar.extractfile('app.py')
>>> f.read()
>>> tar.close()

>>> import tarfile

>>> # create list of files to be archived
>>> file_list = ['app.py', 'config.py', 'CONTRIBUTORS.md', 'tests.py']
>>> with tarfile.open('packages.tar', mode='w') as tar:
...     for file in file_list:
...         tar.add(file)

>>> # Read the contents of the newly created archive
>>> with tarfile.open('package.tar', mode='r') as t:
...     for member in t.getmembers():
...         print(member.name)
app.py
config.py
CONTRIBUTORS.md
tests.py

>>> with tarfile.open('package.tar', mode='a') as tar:
...     tar.add('foo.bar')

>>> with tarfile.open('package.tar', mode='r') as tar:
...     for member in tar.getmembers():
...         print(member.name)
app.py
config.py
CONTRIBUTORS.md
tests.py
foo.bar

>>> files = ['app.py', 'config.py', 'tests.py']
>>> with tarfile.open('packages.tar.gz', mode='w:gz') as tar:
...     tar.add('app.py')
...     tar.add('config.py')
...     tar.add('tests.py')

>>> with tarfile.open('packages.tar.gz', mode='r:gz') as t:
...     for member in t.getmembers():
...         print(member.name)
app.py
config.py
tests.py

shutil.make_archive() takes at least two arguments

• the name of the archive
• an archive format

import shutil

# shutil.make_archive(base_name, format, root_dir)
shutil.make_archive('data/backup', 'tar', 'data/')

shutil.unpack_archive('backup.tar', 'extract_dir/')

import fileinput
for line in fileinput.input()
    process(line)

# File: fileinput-example.py
import fileinput
import sys

files = fileinput.input()
for line in files:
    if fileinput.isfirstline():
        print(f'\n--- Reading {fileinput.filename()} ---')
    print(' -> ' + line, end='')
print()

$ python3 fileinput-example.py bacon.txt cupcake.txt
--- Reading bacon.txt ---
 -> Spicy jalapeno bacon ipsum dolor amet in in aute est qui enim aliquip,
 -> irure cillum drumstick elit.
 -> Doner jowl shank ea exercitation landjaeger incididunt ut porchetta.
 -> Tenderloin bacon aliquip cupidatat chicken chuck quis anim et swine.
 -> Tri-tip doner kevin cillum ham veniam cow hamburger.
 -> Turkey pork loin cupidatat filet mignon capicola brisket cupim ad in.
 -> Ball tip dolor do magna laboris nisi pancetta nostrud doner.

--- Reading cupcake.txt ---
 -> Cupcake ipsum dolor sit amet candy I love cheesecake fruitcake.
 -> Topping muffin cotton candy.
 -> Gummies macaroon jujubes jelly beans marzipan.

• whether or not it is the first line (.isfirstline())
• the line number (.lineno())
• the filename (.filename())