From 665654aff63a50a0812b98ede3e2768ee424c91c Mon Sep 17 00:00:00 2001 From: Cosmin Ciocan Date: Tue, 2 Jan 2024 10:28:43 +0100 Subject: [PATCH 01/46] Add the functionality from v2 by reorganizing compilation into source file groups --- v1/v1.py | 312 ++++++++++++++++++++++++++++++++++++++++++++++++------- 1 file changed, 274 insertions(+), 38 deletions(-) diff --git a/v1/v1.py b/v1/v1.py index ed1aa8e..c4daa5a 100644 --- a/v1/v1.py +++ b/v1/v1.py @@ -1,62 +1,298 @@ +import argparse +import glob import os +import shutil import subprocess import tempfile import uuid +from typing import List, Optional + +from IPython.core.interactiveshell import InteractiveShell +from IPython.core.magic import Magics, cell_magic, line_magic, magics_class -from IPython.core.magic import Magics, cell_magic, magics_class from common import helper -compiler = '/usr/local/cuda/bin/nvcc' -profiler = '/usr/local/cuda/bin/ncu' -ext = '.cu' +DEFAULT_EXEC_FNAME = "cuda_exec.out" +SHARED_GROUP_NAME = "shared" @magics_class class NVCCPlugin(Magics): - - def __init__(self, shell): + def __init__(self, shell: InteractiveShell): super(NVCCPlugin, self).__init__(shell) + self.shell: InteractiveShell # type hint not provided by parent class - self.argparser = helper.get_argparser() + self.parser_cuda = helper.get_parser_cuda() + self.parser_cuda_group_save = helper.get_parser_cuda_group_save() + self.parser_cuda_group_delete = helper.get_parser_cuda_group_delete() + self.parser_cuda_group_run = helper.get_parser_cuda_group_run() - @staticmethod - def compile(file_path): - subprocess.check_output( - [compiler, file_path + ext, "-o", file_path + ".out", '-Wno-deprecated-gpu-targets'], stderr=subprocess.STDOUT) + self.workdir = tempfile.mkdtemp() + print(f'Source files will be saved in "{self.workdir}".') - def run(self, file_path, timeit=False, profile=False, profiler_args=[]): + def _save_source( + self, source_name: str, source_code: str, group_name: str + ) -> None: + """ + Save source code as a .cu or .h file in the group directory where + files can be compiled together. Saving a source file to the group + named "shared" will make those source files available when compiling + any group. + + Args: + source_name: The name of the source file. Must end in ".cu" or + ".h". + source_code: The source code to be written to the source file. + group_name: The name of the group directory where the file will be + saved. + + Raises: + ValueError: If the source name does not have a proper extension. + """ + _, ext = os.path.splitext(source_name) + if ext != ".cu" and ext != ".h": + raise ValueError( + f'Given source name "{source_name}" must end in ".h" or ".cu".' + ) + group_dirpath = os.path.join(self.workdir, group_name) + os.makedirs(group_dirpath, exist_ok=True) + source_fpath = os.path.join(group_dirpath, source_name) + with open(source_fpath, "w", encoding="utf-8") as f: + f.write(source_code) + + def _delete_group(self, group_name: str) -> None: + """ + Removes all source files from the given group. + + Args: + group_name: The name of the source files group. + """ + group_dirpath = os.path.join(self.workdir, group_name) + if os.path.exists(group_dirpath): + shutil.rmtree(group_dirpath) + + def _compile( + self, group_name: str, executable_fname: str = DEFAULT_EXEC_FNAME + ) -> str: + """ + Compiles all source files in a given group together with all source + files from the group named "shared". + + Args: + group_name: The name of the source file group to be compiled. + executable_fname: The output executable file name. Defaults to + "cuda_exec.out". + + Raises: + RuntimeError: If the group does not exist or if does not have any + source files associated with it. + + Returns: + The file path of the resulted executable file. + """ + shared_dirpath = os.path.join(self.workdir, SHARED_GROUP_NAME) + group_dirpath = os.path.join(self.workdir, group_name) + if not os.path.exists(group_dirpath): + raise RuntimeError(f'Group "{group_name}" does not exist.') + + source_files = list(glob.glob(os.path.join(group_dirpath, "*.cu"))) + if len(source_files) == 0: + raise RuntimeError( + f'Group "{group_name}" does not have any source files.' + ) + source_files.extend( + list(glob.glob(os.path.join(shared_dirpath, "*.cu"))) + ) + + executable_fpath = os.path.join(group_dirpath, executable_fname) + + args = [ + "nvcc", + "-I" + shared_dirpath + "," + group_dirpath, + ] + args.extend(source_files) + args.extend( + [ + "-o", + executable_fpath, + "-Wno-deprecated-gpu-targets", + ] + ) + subprocess.check_output(args, stderr=subprocess.STDOUT) + + return executable_fpath + + def _run( + self, + exec_fpath: str, + timeit: bool = False, + profile: bool = False, + profiler_args: str = "", + ) -> str: + """ + Runs a CUDA executable. + + Args: + exec_fpath: The file path of the executable. + timeit: If True, returns the result of the "timeit" magic instead + of the standard output of the CUDA process. Defaults to False. + profile: If True, the executable is profiled with NVIDIA Nsight + Compute profiling tool and its output is added to stdout. + Defaults to False. + profiler_args: The profiler arguments used to customize the + information gathered by it and its overall behaviour. Defaults + to an empty string. + + Returns: + The standard output of the CUDA process or the "timeit" magic + output. + """ if timeit: - stmt = f"subprocess.check_output(['{file_path}.out'], stderr=subprocess.STDOUT)" + stmt = ( + f"subprocess.check_output(['{exec_fpath}']," + " stderr=subprocess.STDOUT)" + ) output = self.shell.run_cell_magic( - magic_name="timeit", line="-q -o import subprocess", cell=stmt) - output = str(output) # convert TimeitResult object to human readable string + magic_name="timeit", line="-q -o import subprocess", cell=stmt + ) + # convert TimeitResult object to human readable string + output = str(output) else: run_args = [] if profile: - run_args.extend([profiler] + profiler_args) - run_args.append(file_path + ".out") - output = subprocess.check_output(run_args, stderr=subprocess.STDOUT) - output = output.decode('utf8') - - helper.print_out(output) - return None + run_args.extend(["ncu"] + profiler_args.split()) + run_args.append(exec_fpath) + output = subprocess.check_output( + run_args, stderr=subprocess.STDOUT + ) + output = output.decode("utf8") + + return output + + def _compile_and_run( + self, group_name: str, args: argparse.Namespace + ) -> str: + try: + exec_fpath = self._compile(group_name) + output = self._run( + exec_fpath=exec_fpath, + timeit=args.timeit, + profile=args.profile, + profiler_args=args.profiler_args, + ) + except subprocess.CalledProcessError as e: + output = e.output.decode("utf8") + return output + + def _read_args( + self, line: str, parser: argparse.ArgumentParser + ) -> Optional[argparse.Namespace]: + """ + Read arguments from the magic line. Makes sure to keep arguments + between double quotes together for use with profiler arguments or + compiler arguments. + + Args: + line: The arguments on the line of the magic call in the jupyter + cell. + parser: The parser which will process the arguments after they are + correctly tokenized. + + Returns: + The parsed arguments. + """ + tokens = line.strip().split('"') + args_tokenized: List[str] = [] + for index, tok in enumerate(tokens): + if index % 2 == 0: + # tokens found outside double quotes are split at whitespace + args_tokenized.extend(tok.split(" ")) + else: + # anything found between double quotes will not be split + args_tokenized.append(tok) + args_tokenized = [arg for arg in args_tokenized if len(arg) > 0] + + try: + return parser.parse_args(args_tokenized) + except SystemExit: + parser.print_help() + return None @cell_magic - def cu(self, line, cell): - try: - args = self.argparser.parse_args(line.split()) - except SystemExit as e: - self.argparser.print_help() + def cuda(self, line: str, cell: str) -> None: + """Compile and run the CUDA code in the cell. + + Args: + line: The arguments on the line of the magic call in the jupyter + cell. + cell: All of the lines in the jupyter cell besides the magic call + itself. It should contain all of the source code to be + compiled and run. + """ + args = self._read_args(line, self.parser_cuda) + if args is None: return - with tempfile.TemporaryDirectory() as tmp_dir: - file_path = os.path.join(tmp_dir, str(uuid.uuid4())) - with open(file_path + ext, "w") as f: - f.write(cell) - try: - self.compile(file_path) - output = self.run(file_path, timeit=args.timeit, profile=args.profile, profiler_args=args.profiler_args) - except subprocess.CalledProcessError as e: - helper.print_out(e.output.decode("utf8")) - output = None - return output + group_name = str(uuid.uuid4()) + self._save_source( + source_name="single_file.cu", + source_code=cell, + group_name=group_name, + ) + + output = self._compile_and_run(group_name, args) + helper.print_out(output) + + @cell_magic + def cuda_group_save(self, line: str, cell: str) -> None: + """ + Save the CUDA code in the cell in a group of source files to be later + compiled and executed by the "cuda_group_run" line magic. + + Args: + line: The arguments on the line of the magic call in the jupyter + cell. + cell: All of the lines in the jupyter cell besides the magic call + itself. It should contain all of the source code to be + saved. + """ + args = self._read_args(line, self.parser_cuda_group_save) + if args is None: + return + + self._save_source( + source_name=args.name, + source_code=cell, + group_name=args.group, + ) + + @line_magic + def cuda_group_run(self, line: str) -> None: + """ + Compile and run all source files inside a specific source file group. + + Args: + line: The arguments on the line of the magic call in the jupyter + cell. + """ + args = self._read_args(line, self.parser_cuda_group_run) + if args is None: + return + + output = self._compile_and_run(args.group, args) + helper.print_out(output) + + @line_magic + def cuda_group_delete(self, line: str) -> None: + """ + Remove all source files inside a specific source file group. + + Args: + line: The arguments on the line of the magic call in the jupyter + cell. + """ + args = self._read_args(line, self.parser_cuda_group_delete) + if args is None: + return + + self._delete_group(args.group) From 0577af2aaea6c4e42a956cfcb12d5d43ffdf4fe2 Mon Sep 17 00:00:00 2001 From: Cosmin Ciocan Date: Tue, 2 Jan 2024 10:29:38 +0100 Subject: [PATCH 02/46] Add argument parsers for the new cell and line magics --- common/helper.py | 116 +++++++++++++++++++++++++++++++++++++++-------- 1 file changed, 98 insertions(+), 18 deletions(-) diff --git a/common/helper.py b/common/helper.py index 17dcfff..94ca898 100644 --- a/common/helper.py +++ b/common/helper.py @@ -1,32 +1,112 @@ import argparse -def get_argparser(): - parser = argparse.ArgumentParser(description='NVCCPlugin params') - parser.add_argument( - '-t', - '--timeit', - action='store_true', - help='If set, returns the output of the "timeit" built-in ipython magic instead of stdout.', +def get_parser_cuda() -> argparse.ArgumentParser: + parser = argparse.ArgumentParser( + description=( + "%%cuda magic that compiles and runs CUDA C++ code in this cell." + ) ) parser.add_argument( - '-p', - '--profile', - action='store_true', - help='If set, runs the nvidia nsight compute profiler. Has no effect if used with --timeit.', + "-t", + "--timeit", + action="store_true", + help=( + 'If set, returns the output of the "timeit" built-in ipython magic' + " instead of stdout." + ), ) parser.add_argument( - '-a', - '--profiler-args', + "-p", + "--profile", + action="store_true", + help=( + "If set, runs the nvidia nsight compute profiler. Has no effect if" + " used with --timeit." + ), + ) + parser.add_argument( + "-a", + "--profiler-args", type=str, - nargs=argparse.REMAINDER, - default=[], - help='Extra options that can be passed to the nvidia nsight compute profiler. ' - 'Must be the last option given to the argument parser so you can pass arguments with dashes.', + default="", + help=( + "Extra options that can be passed to the nvidia nsight compute" + " profiler. Must be the last option given to the argument parser" + " so you can pass arguments with dashes." + ), + ) + return parser + + +def get_parser_cuda_group_run() -> argparse.ArgumentParser: + parser = get_parser_cuda() + parser.description = ( + "%%cuda_group_run magic that compiles and runs source files in a" + " given group." + ) + parser.add_argument( + "-g", + "--group", + type=str, + required=True, + help="The group whose files should be compiled and executed.", + ) + return parser + + +def get_parser_cuda_group_save() -> argparse.ArgumentParser: + parser = argparse.ArgumentParser( + description=( + "%%cuda_group_save magic that saves CUDA C++ code in this cell for" + " later compilation and execution with possibly more source files." + ) + ) + parser.add_argument( + "-n", + "--name", + type=str, + required=True, + help=( + 'The name of the saved source file. Must have either the ".cu" or' + ' ".h" extension. In order to import a header file saved with this' + " magic you can simply add '#include \"\"'." + ), + ) + parser.add_argument( + "-g", + "--group", + type=str, + required=True, + help=( + "The group to which to add the saved source file. Groups are" + " source files that get compiled together and do not interact with" + " other groups. This allows you to have multiple unrelated CUDA" + " programs within the same jupyter notebook. Adding files to a" + ' group named "shared" will make them available to all other' + " source file groups. One use case for this is sharing error" + " handling code." + ), + ) + return parser + + +def get_parser_cuda_group_delete() -> argparse.ArgumentParser: + parser = argparse.ArgumentParser( + description=( + "%%cuda_group_reset magic that deletes all files in a group." + ) + ) + parser.add_argument( + "-g", + "--group", + type=str, + required=True, + help="The group whose files should be deleted.", ) return parser def print_out(out: str): - for l in out.split('\n'): + for l in out.split("\n"): print(l) From 743a46316c412b607c17ded0d2a279aa10cee542 Mon Sep 17 00:00:00 2001 From: Cosmin Ciocan Date: Tue, 2 Jan 2024 10:48:49 +0100 Subject: [PATCH 03/46] Add tests --- tests/conftest.py | 1 + tests/fixtures/__init__.py | 0 tests/fixtures/fixtures.py | 60 +++++++ tests/fixtures/multiple_files/hello.cu | 6 + tests/fixtures/multiple_files/hello.h | 6 + tests/fixtures/multiple_files/main.cu | 6 + tests/fixtures/single_file/hello.cu | 10 ++ tests/test_v1.py | 225 +++++++++++++++++++++++++ 8 files changed, 314 insertions(+) create mode 100644 tests/conftest.py create mode 100644 tests/fixtures/__init__.py create mode 100644 tests/fixtures/fixtures.py create mode 100644 tests/fixtures/multiple_files/hello.cu create mode 100644 tests/fixtures/multiple_files/hello.h create mode 100644 tests/fixtures/multiple_files/main.cu create mode 100644 tests/fixtures/single_file/hello.cu create mode 100644 tests/test_v1.py diff --git a/tests/conftest.py b/tests/conftest.py new file mode 100644 index 0000000..b666e1d --- /dev/null +++ b/tests/conftest.py @@ -0,0 +1 @@ +from fixtures.fixtures import * diff --git a/tests/fixtures/__init__.py b/tests/fixtures/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/tests/fixtures/fixtures.py b/tests/fixtures/fixtures.py new file mode 100644 index 0000000..393a6b9 --- /dev/null +++ b/tests/fixtures/fixtures.py @@ -0,0 +1,60 @@ +import glob +import os +import sys + +sys.path.append(".") + +import pytest +from IPython.core.interactiveshell import InteractiveShell + +from v1.v1 import NVCCPlugin + + +@pytest.fixture(scope="session") +def shell(): + return InteractiveShell() + + +@pytest.fixture(scope="session") +def plugin(shell: InteractiveShell): + return NVCCPlugin(shell=shell) + + +@pytest.fixture(scope="session") +def tests_path(): + return "tests" + + +@pytest.fixture(scope="session") +def fixtures_path(tests_path): + return os.path.join(tests_path, "fixtures") + + +@pytest.fixture(scope="session") +def sample_magic_cu_line(): + # fmt: off + return '--profile --profiler-args "--metrics l1tex__t_sectors_pipe_lsu_mem_global_op_ld.sum"' + # fmt: on + + +@pytest.fixture(scope="session") +def sample_cuda_fpath(fixtures_path: str): + return os.path.join(fixtures_path, "single_file", "hello.cu") + + +@pytest.fixture(scope="session") +def sample_cuda_code(sample_cuda_fpath: str): + with open(sample_cuda_fpath, "r", encoding="utf-8") as f: + return f.read() + + +@pytest.fixture(scope="session") +def timeit_regex(): + return r".+ ± .+ per loop \(mean ± std. dev. of .+ runs, .+ loops each\)" + + +@pytest.fixture(scope="session") +def multiple_source_fpaths(fixtures_path: str): + pattern_h = os.path.join(fixtures_path, "multiple_files", "*.h") + pattern_cu = os.path.join(fixtures_path, "multiple_files", "*.cu") + return list(glob.glob(pattern_h)) + list(glob.glob(pattern_cu)) diff --git a/tests/fixtures/multiple_files/hello.cu b/tests/fixtures/multiple_files/hello.cu new file mode 100644 index 0000000..3b7a67f --- /dev/null +++ b/tests/fixtures/multiple_files/hello.cu @@ -0,0 +1,6 @@ +#include +#include "hello.h" + +__host__ void hello(){ + printf("Hello World!\n"); +} \ No newline at end of file diff --git a/tests/fixtures/multiple_files/hello.h b/tests/fixtures/multiple_files/hello.h new file mode 100644 index 0000000..aaa8cdd --- /dev/null +++ b/tests/fixtures/multiple_files/hello.h @@ -0,0 +1,6 @@ +#ifndef HELLO_H +#define HELLO_H + +void hello(); + +#endif \ No newline at end of file diff --git a/tests/fixtures/multiple_files/main.cu b/tests/fixtures/multiple_files/main.cu new file mode 100644 index 0000000..290472a --- /dev/null +++ b/tests/fixtures/multiple_files/main.cu @@ -0,0 +1,6 @@ +#include "hello.h" + +int main() { + hello(); + return 0; +} \ No newline at end of file diff --git a/tests/fixtures/single_file/hello.cu b/tests/fixtures/single_file/hello.cu new file mode 100644 index 0000000..5620101 --- /dev/null +++ b/tests/fixtures/single_file/hello.cu @@ -0,0 +1,10 @@ +#include + +__host__ void hello(){ + printf("Hello World!\n"); +} + +int main() { + hello(); + return 0; +} \ No newline at end of file diff --git a/tests/test_v1.py b/tests/test_v1.py new file mode 100644 index 0000000..255ffa6 --- /dev/null +++ b/tests/test_v1.py @@ -0,0 +1,225 @@ +import argparse +import math +import os +import re +import shutil +import sys +from typing import List + +import pytest + +sys.path.append(".") + + +from v1.v1 import NVCCPlugin + + +def check_profiler_output(output: str): + # the profiler output will be a line of "Hello World!" along with some + # warning lines which start with "==WARNING==" + lines = output.strip().split("\n") + warn_count = 0 + for line in lines: + if not line.startswith("==WARNING=="): + assert line == "Hello World!" + else: + warn_count += 1 + assert warn_count >= 1 + assert warn_count == len(lines) - 1 + + +def copy_source_to_group( + source_fpath: str, group_name: str, workdir: str +) -> str: + group_dirpath = os.path.join(workdir, group_name) + os.makedirs(group_dirpath, exist_ok=True) + destination_fpath = os.path.join( + group_dirpath, os.path.basename(source_fpath) + ) + shutil.copy(source_fpath, destination_fpath) + return destination_fpath + + +@pytest.fixture(autouse=True, scope="function") +def before_each(plugin: NVCCPlugin): + shutil.rmtree(plugin.workdir, ignore_errors=True) # before test + yield + pass # after test + + +def test_save_source(plugin: NVCCPlugin, sample_cuda_code: str) -> None: + gname = "test_save_source" + sname = "sample.cu" + plugin._save_source(sname, sample_cuda_code, gname) + spath = os.path.join(plugin.workdir, gname, sname) + assert os.path.exists(spath) + with open(spath, "r", encoding="utf-8") as f: + code = f.read() + assert code == sample_cuda_code + + with pytest.raises(ValueError): + plugin._save_source("wrong_extension.txt", sample_cuda_code, gname) + + +def test_delete_group(plugin: NVCCPlugin, sample_cuda_fpath: str) -> None: + gname = "test_delete_group" + source_fpath = copy_source_to_group( + sample_cuda_fpath, gname, plugin.workdir + ) + assert os.path.exists(source_fpath) + plugin._delete_group(gname) + assert not os.path.exists(source_fpath) + + +def test_compile( + plugin: NVCCPlugin, + sample_cuda_fpath: str, +): + # we artificially create a source file group in the plugin workdir + gname = "test_compile" + source_fpath = copy_source_to_group( + sample_cuda_fpath, gname, plugin.workdir + ) + + exec_fpath = plugin._compile(gname) + assert os.path.exists(exec_fpath) + + with pytest.raises(RuntimeError): + plugin._compile("inexistent_group") + + with pytest.raises(RuntimeError): + os.remove(source_fpath) + plugin._compile(gname) + + +def test_run( + plugin: NVCCPlugin, + sample_cuda_fpath: str, +): + gname = "test_run" + copy_source_to_group(sample_cuda_fpath, gname, plugin.workdir) + + exec_fpath = plugin._compile(gname) + output = plugin._run(exec_fpath) + assert output == "Hello World!\n" + + +def test_run_timeit( + plugin: NVCCPlugin, sample_cuda_fpath: str, timeit_regex: str +): + gname = "test_run_timeit" + copy_source_to_group(sample_cuda_fpath, gname, plugin.workdir) + + exec_fpath = plugin._compile(gname) + output = plugin._run(exec_fpath, timeit=True) + assert ( + re.match(timeit_regex, output) is not None + ), f'Output "{output}" does not match the regex "{timeit_regex}".' + + +def test_run_profile(plugin: NVCCPlugin, sample_cuda_fpath: str): + gname = "test_run_profile" + copy_source_to_group(sample_cuda_fpath, gname, plugin.workdir) + + exec_fpath = plugin._compile(gname) + output = plugin._run( + exec_fpath, + profile=True, + # because we are running without a kernel (in the test env we have no + # GPU) it does not matter what arguments we pass to the profiler as its + # output will always be just a few warnings; the reason we add them + # here is to test that no error is produced when passing the arguments + profiler_args=( + "--metrics l1tex__t_sectors_pipe_lsu_mem_global_op_ld.sum" + ), + ) + check_profiler_output(output) + + +def test_compile_and_run_multiple_files( + plugin: NVCCPlugin, multiple_source_fpaths: List[str] +): + """ + Compiles and executes 3 cuda source files from + tests/fixtures/multiple_files. + """ + gname = "test_compile_and_run_multiple_files" + for fpath in multiple_source_fpaths: + copy_source_to_group(fpath, gname, plugin.workdir) + output = plugin._compile_and_run( + gname, argparse.Namespace(timeit=False, profile=True, profiler_args="") + ) + check_profiler_output(output) + + +def test_compile_and_run_multiple_files_shared( + plugin: NVCCPlugin, multiple_source_fpaths: List[str] +): + """ + Compiles and executes 3 cuda source files from + tests/fixtures/multiple_files. However, the hello.cu and hello.h files are + added to the "shared" group which is compiled with all other groups. This + allows sharing error handling code easily and other very common code. + """ + gname = "test_compile_and_run_multiple_files_shared" + for fpath in multiple_source_fpaths: + fname = os.path.basename(fpath) + if fname == "main.cu": + copy_source_to_group(fpath, gname, plugin.workdir) + else: + copy_source_to_group(fpath, "shared", plugin.workdir) + output = plugin._compile_and_run( + gname, argparse.Namespace(timeit=False, profile=True, profiler_args="") + ) + check_profiler_output(output) + + +def test_read_args(plugin: NVCCPlugin): + parser = argparse.ArgumentParser() + parser.add_argument("-a", type=str, required=True) + parser.add_argument("-b", type=float, required=True) + args = plugin._read_args( + '-a "--this has --spaces and --dashes" -b 0.75', parser + ) + assert args.a == "--this has --spaces and --dashes" + assert math.isclose(args.b, 0.75) + + +def test_magic_cuda( + capsys, + plugin: NVCCPlugin, + sample_cuda_code: str, + sample_magic_cu_line: str, +): + plugin.cuda(sample_magic_cu_line, sample_cuda_code) + check_profiler_output(capsys.readouterr().out) + + +def test_magic_cuda_group_save(plugin: NVCCPlugin, sample_cuda_code: str): + gname = "test_save_source" + sname = "sample.cu" + plugin.cuda_group_save(f"-g {gname} -n {sname}", sample_cuda_code) + spath = os.path.join(plugin.workdir, gname, sname) + assert os.path.exists(spath) + with open(spath, "r", encoding="utf-8") as f: + code = f.read() + assert code == sample_cuda_code + + +def test_magic_cuda_group_run( + capsys, plugin: NVCCPlugin, sample_cuda_fpath: str +): + gname = "test_magic_cuda_group_run" + copy_source_to_group(sample_cuda_fpath, gname, plugin.workdir) + plugin.cuda_group_run(f"--group {gname} --profile") + check_profiler_output(capsys.readouterr().out) + + +def test_magic_cuda_group_delete(plugin: NVCCPlugin, sample_cuda_fpath: str): + gname = "test_magic_cuda_group_run" + source_fpath = copy_source_to_group( + sample_cuda_fpath, gname, plugin.workdir + ) + assert os.path.exists(source_fpath) + plugin.cuda_group_delete(f"--group {gname}") + assert not os.path.exists(source_fpath) From ce107de89fdef78eb0dadfec9072203a8153ab5b Mon Sep 17 00:00:00 2001 From: Cosmin Ciocan Date: Tue, 2 Jan 2024 10:52:39 +0100 Subject: [PATCH 04/46] Remove version 2 of the plugin as its functionality was integrated into v1 --- nvcc_plugin.py | 12 +++--- v2/__init__.py | 0 v2/v2.py | 107 ------------------------------------------------- 3 files changed, 5 insertions(+), 114 deletions(-) delete mode 100644 v2/__init__.py delete mode 100644 v2/v2.py diff --git a/nvcc_plugin.py b/nvcc_plugin.py index 81d2c8d..c5afcfe 100644 --- a/nvcc_plugin.py +++ b/nvcc_plugin.py @@ -1,10 +1,8 @@ +from IPython.core.interactiveshell import InteractiveShell + from v1.v1 import NVCCPlugin as NVCC_V1 -from v2.v2 import NVCCPluginV2 as NVCC_V2 -def load_ipython_extension(ip): - nvcc_plugin = NVCC_V1(ip) - ip.register_magics(nvcc_plugin) - - nvcc_plugin_v2 = NVCC_V2(ip) - ip.register_magics(nvcc_plugin_v2) +def load_ipython_extension(shell: InteractiveShell): + nvcc_plugin = NVCC_V1(shell) + shell.register_magics(nvcc_plugin) diff --git a/v2/__init__.py b/v2/__init__.py deleted file mode 100644 index e69de29..0000000 diff --git a/v2/v2.py b/v2/v2.py deleted file mode 100644 index 41511d6..0000000 --- a/v2/v2.py +++ /dev/null @@ -1,107 +0,0 @@ -import os -import subprocess - -from IPython.core.magic import Magics, cell_magic, magics_class -from IPython.core.magic_arguments import argument, magic_arguments, parse_argstring -from common import helper - -compiler = '/usr/local/cuda/bin/nvcc' -profiler = '/usr/local/cuda/bin/ncu' - - -@magics_class -class NVCCPluginV2(Magics): - - def __init__(self, shell): - super(NVCCPluginV2, self).__init__(shell) - self.argparser = helper.get_argparser() - current_dir = os.getcwd() - self.output_dir = os.path.join(current_dir, 'src') - if not os.path.exists(self.output_dir): - os.mkdir(self.output_dir) - print(f'created output directory at {self.output_dir}') - else: - print(f'directory {self.output_dir} already exists') - - self.out = os.path.join(current_dir, "result.out") - print(f'Out bin {self.out}') - - @staticmethod - def compile(output_dir, file_paths, out): - res = subprocess.check_output( - [compiler, '-I' + output_dir, file_paths, "-o", out, '-Wno-deprecated-gpu-targets'], stderr=subprocess.STDOUT) - res = res.decode() - helper.print_out(res) - - def run(self, timeit=False, profile=False, profiler_args=[]): - if timeit: - stmt = f"subprocess.check_output(['{self.out}'], stderr=subprocess.STDOUT)" - output = self.shell.run_cell_magic( - magic_name="timeit", line="-q -o import subprocess", cell=stmt) - output = str(output) # convert TimeitResult object to human readable string - else: - run_args = [] - if profile: - run_args.extend([profiler] + profiler_args) - run_args.append(self.out) - output = subprocess.check_output(run_args, stderr=subprocess.STDOUT) - output = output.decode('utf8') - - helper.print_out(output) - return None - - @magic_arguments() - @argument('-n', '--name', type=str, help='file name that will be produced by the cell. must end with .cu extension') - @argument('-c', '--compile', type=bool, help='Should be compiled?') - @cell_magic - def cuda(self, line='', cell=None): - args = parse_argstring(self.cuda, line) - ex = args.name.split('.')[-1] - if ex not in ['cu', 'h']: - raise Exception('name must end with .cu or .h') - - if not os.path.exists(self.output_dir): - print(f'Output directory does not exist, creating') - try: - os.mkdir(self.output_dir) - except OSError: - print(f"Creation of the directory {self.output_dir} failed") - else: - print(f"Successfully created the directory {self.output_dir}") - - file_path = os.path.join(self.output_dir, args.name) - with open(file_path, "w") as f: - f.write(cell) - - if args.compile: - try: - self.compile(self.output_dir, file_path, self.out) - output = self.run(timeit=args.timeit, profile=args.profile, profiler_args=args.profiler_args) - except subprocess.CalledProcessError as e: - helper.print_out(e.output.decode("utf8")) - output = None - else: - output = f'File written in {file_path}' - - return output - - @cell_magic - def cuda_run(self, line='', cell=None): - try: - args = self.argparser.parse_args(line.split()) - except SystemExit: - self.argparser.print_help() - return - - try: - cuda_src = os.listdir(self.output_dir) - cuda_src = [os.path.join(self.output_dir, x) - for x in cuda_src if x[-3:] == '.cu'] - print(f'found sources: {cuda_src}') - self.compile(self.output_dir, ' '.join(cuda_src), self.out) - output = self.run(timeit=args.timeit, profile=args.profile, profiler_args=args.profiler_args) - except subprocess.CalledProcessError as e: - helper.print_out(e.output.decode("utf8")) - output = None - - return output From 11146a3e10857f2234fa72cf0ddcf2f7df83a1dd Mon Sep 17 00:00:00 2001 From: Cosmin Ciocan Date: Tue, 2 Jan 2024 10:59:08 +0100 Subject: [PATCH 05/46] Move source files to nvcc4jupyter directory --- common/__init__.py | 0 nvcc4jupyter/__init__.py | 1 + common/helper.py => nvcc4jupyter/parsers.py | 4 ---- v1/v1.py => nvcc4jupyter/plugin.py | 19 ++++++++++++------- v1/__init__.py | 0 5 files changed, 13 insertions(+), 11 deletions(-) delete mode 100644 common/__init__.py create mode 100644 nvcc4jupyter/__init__.py rename common/helper.py => nvcc4jupyter/parsers.py (97%) rename v1/v1.py => nvcc4jupyter/plugin.py (95%) delete mode 100644 v1/__init__.py diff --git a/common/__init__.py b/common/__init__.py deleted file mode 100644 index e69de29..0000000 diff --git a/nvcc4jupyter/__init__.py b/nvcc4jupyter/__init__.py new file mode 100644 index 0000000..5becc17 --- /dev/null +++ b/nvcc4jupyter/__init__.py @@ -0,0 +1 @@ +__version__ = "1.0.0" diff --git a/common/helper.py b/nvcc4jupyter/parsers.py similarity index 97% rename from common/helper.py rename to nvcc4jupyter/parsers.py index 94ca898..62a1c15 100644 --- a/common/helper.py +++ b/nvcc4jupyter/parsers.py @@ -106,7 +106,3 @@ def get_parser_cuda_group_delete() -> argparse.ArgumentParser: ) return parser - -def print_out(out: str): - for l in out.split("\n"): - print(l) diff --git a/v1/v1.py b/nvcc4jupyter/plugin.py similarity index 95% rename from v1/v1.py rename to nvcc4jupyter/plugin.py index c4daa5a..f5b77d4 100644 --- a/v1/v1.py +++ b/nvcc4jupyter/plugin.py @@ -10,22 +10,27 @@ from typing import List, Optional from IPython.core.interactiveshell import InteractiveShell from IPython.core.magic import Magics, cell_magic, line_magic, magics_class -from common import helper +from . import parsers DEFAULT_EXEC_FNAME = "cuda_exec.out" SHARED_GROUP_NAME = "shared" +def print_out(out: str): + for l in out.split("\n"): + print(l) + + @magics_class class NVCCPlugin(Magics): def __init__(self, shell: InteractiveShell): super(NVCCPlugin, self).__init__(shell) self.shell: InteractiveShell # type hint not provided by parent class - self.parser_cuda = helper.get_parser_cuda() - self.parser_cuda_group_save = helper.get_parser_cuda_group_save() - self.parser_cuda_group_delete = helper.get_parser_cuda_group_delete() - self.parser_cuda_group_run = helper.get_parser_cuda_group_run() + self.parser_cuda = parsers.get_parser_cuda() + self.parser_cuda_group_save = parsers.get_parser_cuda_group_save() + self.parser_cuda_group_delete = parsers.get_parser_cuda_group_delete() + self.parser_cuda_group_run = parsers.get_parser_cuda_group_run() self.workdir = tempfile.mkdtemp() print(f'Source files will be saved in "{self.workdir}".') @@ -241,7 +246,7 @@ class NVCCPlugin(Magics): ) output = self._compile_and_run(group_name, args) - helper.print_out(output) + print_out(output) @cell_magic def cuda_group_save(self, line: str, cell: str) -> None: @@ -280,7 +285,7 @@ class NVCCPlugin(Magics): return output = self._compile_and_run(args.group, args) - helper.print_out(output) + print_out(output) @line_magic def cuda_group_delete(self, line: str) -> None: diff --git a/v1/__init__.py b/v1/__init__.py deleted file mode 100644 index e69de29..0000000 From 7d8e04622c98615cab1e084b06f057df48c236e1 Mon Sep 17 00:00:00 2001 From: Cosmin Ciocan Date: Tue, 2 Jan 2024 10:59:53 +0100 Subject: [PATCH 06/46] Update tests imports --- tests/fixtures/fixtures.py | 2 +- tests/test_v1.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/fixtures/fixtures.py b/tests/fixtures/fixtures.py index 393a6b9..f898df6 100644 --- a/tests/fixtures/fixtures.py +++ b/tests/fixtures/fixtures.py @@ -7,7 +7,7 @@ sys.path.append(".") import pytest from IPython.core.interactiveshell import InteractiveShell -from v1.v1 import NVCCPlugin +from nvcc4jupyter.plugin import NVCCPlugin @pytest.fixture(scope="session") diff --git a/tests/test_v1.py b/tests/test_v1.py index 255ffa6..45f5d4c 100644 --- a/tests/test_v1.py +++ b/tests/test_v1.py @@ -11,7 +11,7 @@ import pytest sys.path.append(".") -from v1.v1 import NVCCPlugin +from nvcc4jupyter.plugin import NVCCPlugin def check_profiler_output(output: str): From 2a5a7ffeb7f86416121dc6183e1d8ba9523496ce Mon Sep 17 00:00:00 2001 From: Cosmin Ciocan Date: Tue, 2 Jan 2024 11:16:26 +0100 Subject: [PATCH 07/46] Use pyproject.toml to conform with PEP 621 --- pyproject.toml | 297 +++++++++++++++++++++++++++++++++++++ setup.py | 13 -- tests/fixtures/fixtures.py | 3 - tests/test_v1.py | 8 +- 4 files changed, 299 insertions(+), 22 deletions(-) create mode 100644 pyproject.toml delete mode 100644 setup.py diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 0000000..2c795eb --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,297 @@ +[build-system] +requires = ["hatchling >= 1.13.0"] +build-backend = "hatchling.build" + +[project] +name = "nvcc4jupyter" +description = "Jupyter notebook plugin to run CUDA C/C++ code" +readme = "README.md" +requires-python = ">=3.7" +license = {text = "MIT License"} +authors = [ + { name = "Andrei Nechaev", email = "lyfaradey@yahoo.com" }, +] +classifiers = [ + "Programming Language :: Python :: 3", + "Programming Language :: Python", +] +dependencies = [] +dynamic = ["version"] + +[tool.hatch.version] +path = "nvcc4jupyter/__init__.py" + +[tool.hatch.build.targets.wheel] +packages = ["nvcc4jupyter"] + +[project.optional-dependencies] +testing = ["pytest>=7.4.3", "IPython>=8.19.0"] + + +[tool.pytest.ini_options] +addopts = [ + "--color=yes", + "--durations=0", + "--strict-markers", + "--doctest-modules", +] +filterwarnings = [ + "ignore::DeprecationWarning", + "ignore::UserWarning", +] +log_cli = "True" +markers = [ + "slow: slow tests", +] +minversion = "6.0" +testpaths = "tests/" + +[tool.coverage.report] +exclude_lines = [ + "pragma: nocover", + "raise NotImplementedError", + "raise NotImplementedError()", + "if __name__ == .__main__.:", +] + +[tool.isort] +profile = "black" + +[tool.bandit] +exclude_dirs = ["build","dist","tests","scripts"] +number = 4 +recursive = true +targets = "src" +skips = ["B101", "B311"] + +[tool.black] +line-length = 79 +fast = true +experimental-string-processing = true + +[tool.coverage.run] +branch = true + +[tool.flake8] +max-line-length = 79 +select = "F,E,W,B,B901,B902,B903" +exclude = [ + ".eggs", + ".git", + ".tox", + "nssm", + "obj", + "out", + "packages", + "pywin32", + "tests", + "swagger_client" +] +ignore = [ + "E722", + "B001", + "W503", + "E203" +] + +[tool.pyright] +include = ["src"] +exclude = [ + "**/node_modules", + "**/__pycache__", +] +venv = "env37" + +reportMissingImports = true +reportMissingTypeStubs = false + +pythonVersion = "3.7" +pythonPlatform = "Linux" + +executionEnvironments = [ + { root = "src" } +] + +[tool.tox] +legacy_tox_ini = """ +[tox] +envlist = py, integration, spark, all +[testenv] +commands = + pytest -m "not integration and not spark" {posargs} +[testenv:integration] +commands = + pytest -m "integration" {posargs} +[testenv:spark] +extras = spark +setenv = + PYSPARK_DRIVER_PYTHON = {envpython} + PYSPARK_PYTHON = {envpython} +commands = + pytest -m "spark" {posargs} +[testenv:all] +extras = all +setenv = + PYSPARK_DRIVER_PYTHON = {envpython} + PYSPARK_PYTHON = {envpython} +commands = + pytest {posargs} +""" + +[tool.pylint] +extension-pkg-whitelist= [ + "numpy", + "torch", + "cv2", + "pyodbc", + "pydantic", + "ciso8601", + "netcdf4", + "scipy" +] +ignore="CVS" +ignore-patterns="test.*?py,conftest.py" +init-hook='import sys; sys.setrecursionlimit(8 * sys.getrecursionlimit())' +jobs=0 +limit-inference-results=100 +persistent="yes" +suggestion-mode="yes" +unsafe-load-any-extension="no" + +[tool.pylint.'MESSAGES CONTROL'] +enable="c-extension-no-member" + +[tool.pylint.'REPORTS'] +evaluation="10.0 - ((float(5 * error + warning + refactor + convention) / statement) * 10)" +output-format="text" +reports="no" +score="yes" + +[tool.pylint.'REFACTORING'] +max-nested-blocks=5 +never-returning-functions="sys.exit" + +[tool.pylint.'BASIC'] +argument-naming-style="snake_case" +attr-naming-style="snake_case" +bad-names= [ + "foo", + "bar" +] +class-attribute-naming-style="any" +class-naming-style="PascalCase" +const-naming-style="UPPER_CASE" +docstring-min-length=-1 +function-naming-style="snake_case" +good-names= [ + "i", + "j", + "k", + "ex", + "Run", + "_" +] +include-naming-hint="yes" +inlinevar-naming-style="any" +method-naming-style="snake_case" +module-naming-style="any" +no-docstring-rgx="^_" +property-classes="abc.abstractproperty" +variable-naming-style="snake_case" + +[tool.pylint.'FORMAT'] +ignore-long-lines="^\\s*(# )?.*['\"]??" +indent-after-paren=4 +indent-string=' ' +max-line-length=79 +max-module-lines=1000 +single-line-class-stmt="no" +single-line-if-stmt="no" + +[tool.pylint.'LOGGING'] +logging-format-style="old" +logging-modules="logging" + +[tool.pylint.'MISCELLANEOUS'] +notes= [ + "FIXME", + "XXX", + "TODO" +] + +[tool.pylint.'SIMILARITIES'] +ignore-comments="yes" +ignore-docstrings="yes" +ignore-imports="yes" +min-similarity-lines=7 + +[tool.pylint.'SPELLING'] +max-spelling-suggestions=4 +spelling-store-unknown-words="no" + +[tool.pylint.'STRING'] +check-str-concat-over-line-jumps="no" + +[tool.pylint.'TYPECHECK'] +contextmanager-decorators="contextlib.contextmanager" +generated-members="numpy.*,np.*,pyspark.sql.functions,collect_list" +ignore-mixin-members="yes" +ignore-none="yes" +ignore-on-opaque-inference="yes" +ignored-classes="optparse.Values,thread._local,_thread._local,numpy,torch,swagger_client" +ignored-modules="numpy,torch,swagger_client,netCDF4,scipy" +missing-member-hint="yes" +missing-member-hint-distance=1 +missing-member-max-choices=1 + +[tool.pylint.'VARIABLES'] +additional-builtins="dbutils" +allow-global-unused-variables="yes" +callbacks= [ + "cb_", + "_cb" +] +dummy-variables-rgx="_+$|(_[a-zA-Z0-9_]*[a-zA-Z0-9]+?$)|dummy|^ignored_|^unused_" +ignored-argument-names="_.*|^ignored_|^unused_" +init-import="no" +redefining-builtins-modules="six.moves,past.builtins,future.builtins,builtins,io" + +[tool.pylint.'CLASSES'] +defining-attr-methods= [ + "__init__", + "__new__", + "setUp", + "__post_init__" +] +exclude-protected= [ + "_asdict", + "_fields", + "_replace", + "_source", + "_make" +] +valid-classmethod-first-arg="cls" +valid-metaclass-classmethod-first-arg="cls" + +[tool.pylint.'DESIGN'] +max-args=5 +max-attributes=7 +max-bool-expr=5 +max-branches=12 +max-locals=15 +max-parents=7 +max-public-methods=20 +max-returns=6 +max-statements=50 +min-public-methods=2 + +[tool.pylint.'IMPORTS'] +allow-wildcard-with-all="no" +analyse-fallback-blocks="no" +deprecated-modules="optparse,tkinter.tix" + +[tool.pylint.'EXCEPTIONS'] +overgeneral-exceptions= [ + "BaseException", + "Exception" +] \ No newline at end of file diff --git a/setup.py b/setup.py deleted file mode 100644 index 0643352..0000000 --- a/setup.py +++ /dev/null @@ -1,13 +0,0 @@ -from distutils.core import setup - -setup( - name='NVCCPlugin', - version='0.0.2', - author='Andrei Nechaev', - author_email='lyfaradey@yahoo.com', - py_modules=['nvcc_plugin', 'v2.v2', 'v1.v1', 'common.helper'], - url='https://github.com/andreinechaev/nvcc4jupyter', - license='LICENSE', - description='Jupyter notebook plugin to run CUDA C/C++ code', - # long_description=open('README.md').read(), -) diff --git a/tests/fixtures/fixtures.py b/tests/fixtures/fixtures.py index f898df6..f97b086 100644 --- a/tests/fixtures/fixtures.py +++ b/tests/fixtures/fixtures.py @@ -1,8 +1,5 @@ import glob import os -import sys - -sys.path.append(".") import pytest from IPython.core.interactiveshell import InteractiveShell diff --git a/tests/test_v1.py b/tests/test_v1.py index 45f5d4c..05d340e 100644 --- a/tests/test_v1.py +++ b/tests/test_v1.py @@ -3,14 +3,10 @@ import math import os import re import shutil -import sys from typing import List import pytest -sys.path.append(".") - - from nvcc4jupyter.plugin import NVCCPlugin @@ -42,9 +38,9 @@ def copy_source_to_group( @pytest.fixture(autouse=True, scope="function") def before_each(plugin: NVCCPlugin): - shutil.rmtree(plugin.workdir, ignore_errors=True) # before test + shutil.rmtree(plugin.workdir, ignore_errors=True) # before test yield - pass # after test + pass # after test def test_save_source(plugin: NVCCPlugin, sample_cuda_code: str) -> None: From 99e45d93b28548e0a2a51605abe0fac1ed0d0641 Mon Sep 17 00:00:00 2001 From: Cosmin Ciocan Date: Tue, 2 Jan 2024 11:18:05 +0100 Subject: [PATCH 08/46] Add vscode configs for black, pylint, flake and isort extensions --- .vscode/settings.json | 32 ++++++++++++++++++++++++++++++++ 1 file changed, 32 insertions(+) create mode 100644 .vscode/settings.json diff --git a/.vscode/settings.json b/.vscode/settings.json new file mode 100644 index 0000000..0e16f80 --- /dev/null +++ b/.vscode/settings.json @@ -0,0 +1,32 @@ +{ + "editor.formatOnSave": true, + "editor.formatOnPaste": true, + "files.trimTrailingWhitespace": true, + "files.autoSave": "onFocusChange", + "git.autofetch": true, + "[jsonc]": { + "editor.defaultFormatter": "vscode.json-language-features" + }, + "[python]": { + "editor.defaultFormatter": "ms-python.black-formatter", + "editor.formatOnSave": true, + "editor.codeActionsOnSave": { + "source.organizeImports": "explicit" + }, + }, + "python.defaultInterpreterPath": "/usr/local/bin/python", + "python.testing.unittestEnabled": false, + "python.testing.pytestEnabled": true, + "pylint.args": [ + "--rcfile=pyproject.toml" + ], + "black-formatter.args": [ + "--config=pyproject.toml" + ], + "flake8.args": [ + "--toml-config=pyproject.toml" + ], + "isort.args": [ + "--settings-path=pyproject.toml" + ] +} \ No newline at end of file From d3e68f53e07324600e8f0aea5f8babffb382cf9b Mon Sep 17 00:00:00 2001 From: Cosmin Ciocan Date: Tue, 2 Jan 2024 11:19:53 +0100 Subject: [PATCH 09/46] Move load_ipython_extension function to plugin.py --- nvcc4jupyter/plugin.py | 5 +++++ nvcc_plugin.py | 8 -------- 2 files changed, 5 insertions(+), 8 deletions(-) delete mode 100644 nvcc_plugin.py diff --git a/nvcc4jupyter/plugin.py b/nvcc4jupyter/plugin.py index f5b77d4..e278d56 100644 --- a/nvcc4jupyter/plugin.py +++ b/nvcc4jupyter/plugin.py @@ -301,3 +301,8 @@ class NVCCPlugin(Magics): return self._delete_group(args.group) + + +def load_ipython_extension(shell: InteractiveShell): + nvcc_plugin = NVCCPlugin(shell) + shell.register_magics(nvcc_plugin) diff --git a/nvcc_plugin.py b/nvcc_plugin.py deleted file mode 100644 index c5afcfe..0000000 --- a/nvcc_plugin.py +++ /dev/null @@ -1,8 +0,0 @@ -from IPython.core.interactiveshell import InteractiveShell - -from v1.v1 import NVCCPlugin as NVCC_V1 - - -def load_ipython_extension(shell: InteractiveShell): - nvcc_plugin = NVCC_V1(shell) - shell.register_magics(nvcc_plugin) From 56601ca6bd6d30e69e299c36866902cf4a7487c4 Mon Sep 17 00:00:00 2001 From: Cosmin Ciocan Date: Tue, 2 Jan 2024 11:21:59 +0100 Subject: [PATCH 10/46] Add usual python .gitignore items --- .gitignore | 37 ++++++++++++++++++++++++++++++++++++- 1 file changed, 36 insertions(+), 1 deletion(-) diff --git a/.gitignore b/.gitignore index 3d72576..3485fef 100644 --- a/.gitignore +++ b/.gitignore @@ -1,2 +1,37 @@ +# Byte-compiled / optimized / DLL files +__pycache__/ +*.py[cod] + +# Distribution / packaging +bin/ +build/ +develop-eggs/ +dist/ +eggs/ +lib/ +lib64/ +parts/ +sdist/ +var/ +*.egg-info/ +.installed.cfg +*.egg + +# Installer logs +pip-log.txt +pip-delete-this-directory.txt + +# Unit test / coverage reports +.tox/ +.coverage +.cache +nosetests.xml +coverage.xml + +# Virtual Environment +*env* + +# Misc +.pytest_cache/ .DS_Store -.idea \ No newline at end of file +.idea From fbffb60960f2ad95f55f21510e4edd9b610a066e Mon Sep 17 00:00:00 2001 From: Cosmin Ciocan Date: Tue, 2 Jan 2024 11:34:27 +0100 Subject: [PATCH 11/46] Rename test file --- tests/{test_v1.py => test_plugin.py} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename tests/{test_v1.py => test_plugin.py} (100%) diff --git a/tests/test_v1.py b/tests/test_plugin.py similarity index 100% rename from tests/test_v1.py rename to tests/test_plugin.py From 8ae548362978997f11348ff4c37603f4733d5ab6 Mon Sep 17 00:00:00 2001 From: Cosmin Ciocan Date: Tue, 2 Jan 2024 15:24:41 +0100 Subject: [PATCH 12/46] Add testing github workflow --- .github/workflows/test.yml | 41 ++++++++++++++++++++++++++++++++++++++ tests/__init__.py | 0 tests/conftest.py | 2 +- tests/requirements.txt | 2 ++ 4 files changed, 44 insertions(+), 1 deletion(-) create mode 100644 .github/workflows/test.yml create mode 100644 tests/__init__.py create mode 100644 tests/requirements.txt diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml new file mode 100644 index 0000000..9d669a0 --- /dev/null +++ b/.github/workflows/test.yml @@ -0,0 +1,41 @@ +name: Tests + +on: + push: + branches: [master] + pull_request: + branches: [master, "release/*", "dev"] + +jobs: + run_tests_ubuntu: + runs-on: ${{ matrix.os }} + + strategy: + fail-fast: false + matrix: + os: ["ubuntu-latest"] + python-version: ["3.8", "3.9", "3.10"] + + timeout-minutes: 20 + + steps: + - name: Checkout + uses: actions/checkout@v3 + + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v3 + with: + python-version: ${{ matrix.python-version }} + + - name: Install dependencies + run: | + python -m pip install --upgrade pip + pip install -r tests/requirements.txt + + - name: List dependencies + run: | + python -m pip list + + - name: Run pytest + run: | + pytest -v diff --git a/tests/__init__.py b/tests/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/tests/conftest.py b/tests/conftest.py index b666e1d..634fb8c 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -1 +1 @@ -from fixtures.fixtures import * +from .fixtures.fixtures import * diff --git a/tests/requirements.txt b/tests/requirements.txt new file mode 100644 index 0000000..78d9bb7 --- /dev/null +++ b/tests/requirements.txt @@ -0,0 +1,2 @@ +pytest>=7.4.3 +IPython>=8.19.0 \ No newline at end of file From f467060bcf13bb577f3e0ccbe6bd32e59546e515 Mon Sep 17 00:00:00 2001 From: Cosmin Ciocan Date: Tue, 2 Jan 2024 15:26:46 +0100 Subject: [PATCH 13/46] Install nvidia toolkit for testing --- .github/workflows/test.yml | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 9d669a0..8a87fb6 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -27,7 +27,12 @@ jobs: with: python-version: ${{ matrix.python-version }} - - name: Install dependencies + - name: Install CUDA tools + run: | + apt update + apt install nvidia-cuda-toolkit + + - name: Install Python dependencies run: | python -m pip install --upgrade pip pip install -r tests/requirements.txt From 5f1bc886259a10888bd8f2122d2da4cf94805f93 Mon Sep 17 00:00:00 2001 From: Cosmin Ciocan Date: Tue, 2 Jan 2024 15:27:46 +0100 Subject: [PATCH 14/46] Add sudo to apt commands --- .github/workflows/test.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 8a87fb6..068b773 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -29,8 +29,8 @@ jobs: - name: Install CUDA tools run: | - apt update - apt install nvidia-cuda-toolkit + sudo apt update + sudo apt install nvidia-cuda-toolkit - name: Install Python dependencies run: | From d70fe0f5e2eee07cd576bbc273b38d3d6b0ddc0b Mon Sep 17 00:00:00 2001 From: Cosmin Ciocan Date: Tue, 2 Jan 2024 15:35:42 +0100 Subject: [PATCH 15/46] Create a directory to bypass a profiler error to be able to test with host code only --- .github/workflows/test.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 068b773..2ec1379 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -27,10 +27,14 @@ jobs: with: python-version: ${{ matrix.python-version }} + # the mkdir command bypasses a profiler error, which allows us to run it + # with host code only to at least check that the profiler parameters are + # correctly provided - name: Install CUDA tools run: | sudo apt update sudo apt install nvidia-cuda-toolkit + mkdir -p /usr/lib/x86_64-linux-gnu/nsight-compute/sections - name: Install Python dependencies run: | From 4ade0eac9fe3f672da850d48be52382d4a1f9aaf Mon Sep 17 00:00:00 2001 From: Cosmin Ciocan Date: Tue, 2 Jan 2024 15:41:39 +0100 Subject: [PATCH 16/46] Change test versions to match newest IPython and add sudo to mkdir --- .github/workflows/test.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 2ec1379..30af8a2 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -14,7 +14,7 @@ jobs: fail-fast: false matrix: os: ["ubuntu-latest"] - python-version: ["3.8", "3.9", "3.10"] + python-version: ["3.10", "3.11", "3.12"] timeout-minutes: 20 @@ -34,7 +34,7 @@ jobs: run: | sudo apt update sudo apt install nvidia-cuda-toolkit - mkdir -p /usr/lib/x86_64-linux-gnu/nsight-compute/sections + sudo mkdir -p /usr/lib/x86_64-linux-gnu/nsight-compute/sections - name: Install Python dependencies run: | From 6db8bf2b525cbd9e31931bab9bb708a5ae15ec55 Mon Sep 17 00:00:00 2001 From: Cosmin Ciocan Date: Fri, 5 Jan 2024 13:41:29 +0100 Subject: [PATCH 17/46] Add pypi publishing workflow --- .github/workflows/publish-to-pypi.yml | 46 +++++++++++++++++++++++++++ 1 file changed, 46 insertions(+) create mode 100644 .github/workflows/publish-to-pypi.yml diff --git a/.github/workflows/publish-to-pypi.yml b/.github/workflows/publish-to-pypi.yml new file mode 100644 index 0000000..a92716f --- /dev/null +++ b/.github/workflows/publish-to-pypi.yml @@ -0,0 +1,46 @@ +name: Publish Python 🐍 distribution 📦 to PyPI + +on: push + +jobs: + build: + name: Build distribution 📦 + runs-on: ubuntu-latest + + steps: + - uses: actions/checkout@v4 + - name: Set up Python + uses: actions/setup-python@v4 + with: + python-version: "3.x" + - name: Install pypa/build + run: python3 -m pip install build --user + - name: Build a binary wheel and a source tarball + run: python3 -m build + - name: Store the distribution packages + uses: actions/upload-artifact@v3 + with: + name: python-package-distributions + path: dist/ + + publish-to-pypi: + name: >- + Publish Python 🐍 distribution 📦 to PyPI + if: startsWith(github.ref, 'refs/tags/') # only publish to PyPI on tag pushes + needs: + - build + runs-on: ubuntu-latest + environment: + name: pypi + url: https://pypi.org/p/nvcc4jupyter + permissions: + id-token: write + + steps: + - name: Download all the dists + uses: actions/download-artifact@v3 + with: + name: python-package-distributions + path: dist/ + - name: Publish distribution 📦 to PyPI + uses: pypa/gh-action-pypi-publish@release/v1 From 3a0d40dab91ca1e598ca48888cd6d9a36cb3493b Mon Sep 17 00:00:00 2001 From: Cosmin Ciocan Date: Fri, 5 Jan 2024 14:15:22 +0100 Subject: [PATCH 18/46] Add load_ipython_extension function to package root --- nvcc4jupyter/__init__.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/nvcc4jupyter/__init__.py b/nvcc4jupyter/__init__.py index 5becc17..8326385 100644 --- a/nvcc4jupyter/__init__.py +++ b/nvcc4jupyter/__init__.py @@ -1 +1,3 @@ +from .plugin import NVCCPlugin, load_ipython_extension + __version__ = "1.0.0" From b6c38ea55a52115f3969d9c608984e29eb0e814d Mon Sep 17 00:00:00 2001 From: Cosmin Ciocan Date: Fri, 5 Jan 2024 14:46:55 +0100 Subject: [PATCH 19/46] Add python tested versions in pyproject.toml --- pyproject.toml | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index 2c795eb..3a99a3a 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -12,8 +12,11 @@ authors = [ { name = "Andrei Nechaev", email = "lyfaradey@yahoo.com" }, ] classifiers = [ - "Programming Language :: Python :: 3", "Programming Language :: Python", + "Programming Language :: Python :: 3", + 'Programming Language :: Python :: 3.10', + 'Programming Language :: Python :: 3.11', + 'Programming Language :: Python :: 3.12', ] dependencies = [] dynamic = ["version"] From 8b88c5dc3492e892b9f03cef335bb33bc9207404 Mon Sep 17 00:00:00 2001 From: Cosmin Ciocan Date: Fri, 5 Jan 2024 15:17:55 +0100 Subject: [PATCH 20/46] Add code coverage job in testing workflow --- .github/workflows/test.yml | 36 ++++++++++++++++++++++++++++++++++++ 1 file changed, 36 insertions(+) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 30af8a2..6fd78e1 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -48,3 +48,39 @@ jobs: - name: Run pytest run: | pytest -v + + # upload code coverage report + code-coverage: + runs-on: ubuntu-latest + + steps: + - name: Checkout + uses: actions/checkout@v2 + with: + lfs: "true" + - run: git lfs pull + + - name: Set up Python 3.10 + uses: actions/setup-python@v2 + with: + python-version: "3.10" + + - name: Install CUDA tools + run: | + sudo apt update + sudo apt install nvidia-cuda-toolkit + sudo mkdir -p /usr/lib/x86_64-linux-gnu/nsight-compute/sections + + - name: Install Python dependencies + run: | + python -m pip install --upgrade pip + pip install -r tests/requirements.txt + pip install pytest-cov[toml] + + - name: Run tests and collect coverage + run: pytest --cov nvcc4jupyter + + - name: Upload coverage to Codecov + uses: codecov/codecov-action@v3 + env: + CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }} From 6a02223f96dd7260340202feadfcc8642a3c3bbb Mon Sep 17 00:00:00 2001 From: Cosmin Ciocan Date: Fri, 5 Jan 2024 15:58:34 +0100 Subject: [PATCH 21/46] Update readme to include useful badges --- README.md | 84 ++++++++++++++++++++++++++++++++++++------------------- 1 file changed, 55 insertions(+), 29 deletions(-) diff --git a/README.md b/README.md index b9ddce0..5e34ef3 100644 --- a/README.md +++ b/README.md @@ -1,37 +1,63 @@ -## NVCC Plugin for Jupyter notebook +# nvcc4jupyter: CUDA C++ plugin for Jupyter Notebook -### V2 is available +| | | +| --- | --- | +| Testing | ![Python Versions][python-version] [![CI - Test][test-badge]][test-workflow] [![Coverage][coverage-badge]][coverage-results] | +| Package | [![PyPI Latest Release][pypi-latest-version]][pypi-project-url] [![PyPI Downloads][pypi-downloads]][pypi-project-url] | -V2 brings support of multiple source and header files. -##### Usage +[python-version]: https://img.shields.io/pypi/pyversions/nvcc4jupyter +[test-badge]: https://github.com/cosminc98/nvcc4jupyter/actions/workflows/test.yml/badge.svg +[test-workflow]: https://github.com/cosminc98/nvcc4jupyter/actions/workflows/test.yml +[coverage-badge]: https://codecov.io/github/cosminc98/nvcc4jupyter/coverage.svg?branch=master +[coverage-results]: https://codecov.io/gh/cosminc98/nvcc4jupyter +[pypi-project-url]: https://pypi.org/project/nvcc4jupyter/ +[pypi-latest-version]: https://img.shields.io/pypi/v/nvcc4jupyter.svg +[pypi-downloads]: https://img.shields.io/pypi/dd/nvcc4jupyter.svg?label=PyPI%20downloads -- Install and load extension -``` -!pip install git+https://github.com/andreinechaev/nvcc4jupyter.git -%load_ext nvcc_plugin + +## What is it? + +**nvcc4jupyter** is a Jupyter Notebook plugin that provides cell and line +[magics](https://ipython.readthedocs.io/en/stable/interactive/magics.html) +to allow running CUDA C++ code from a notebook. This is especially +useful when combined with a hosted service such a Google's +[Colab](https://colab.research.google.com/) which provide CUDA capable GPUs +and you can start learning CUDA C++ without having to install anything or even +to own a GPU yourself. + +## Table of Contents + +- [Main Features](#main-features) +- [Install](#install) +- [Dependencies](#dependencies) +- [Installation from sources](#installation-from-sources) +- [License](#license) +- [Documentation](#documentation) + +## Main Features +Here are just a few of the things that nvcc4jupyter does well: + + - TODO1 + - TODO2 + +## Install +The installer for the latest released version is available at the [Python +Package Index (PyPI)](https://pypi.org/project/nvcc4jupyter). + +```sh +pip install nvcc4jupyter ``` -- Mark a cell to be treated as cuda cell -> `%%cuda --name example.cu --compile false` ->> NOTE: The cell must contain either code or comments to be run successfully. ->> It accepts 2 arguments. `-n` | `--name` - which is the name of either CUDA source or Header ->> The name parameter must have extension `.cu` or `.h` ->> Second argument `-c` | `--compile`; default value is `false`. The argument is a flag to specify ->> if the cell will be compiled and run right away or not. It might be usefull if you're playing in ->> the `main` function +## Usage +TODO -- To compile and run all CUDA files you need to run -``` -%%cuda_run -# This line just to bypass an exeption and can contain any text -``` +## Documentation +The official documentation is hosted on [TODO](TODO). -- To profile your CUDA kernels using NVIDIA Nsight Compute CLI profiler you need to run -``` -%%cu --profile -``` -- You can add options to the profiler. Keep in mind that any argument after "--profiler-args" will be considered as a profiler argument. For example, to select which sections to collect metrics for you need to run -``` -%%cu --profile --profiler-args --section SpeedOfLight --section MemoryWorkloadAnalysis --section Occupancy -``` +## License +[TODO](LICENSE) + +
+ +[Go to Top](#table-of-contents) From c6ab2dce209a097f4592fe4696229a89193a9277 Mon Sep 17 00:00:00 2001 From: Cosmin Ciocan Date: Fri, 5 Jan 2024 15:59:15 +0100 Subject: [PATCH 22/46] Update version to 1.0.1 --- nvcc4jupyter/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nvcc4jupyter/__init__.py b/nvcc4jupyter/__init__.py index 8326385..08a8aff 100644 --- a/nvcc4jupyter/__init__.py +++ b/nvcc4jupyter/__init__.py @@ -1,3 +1,3 @@ from .plugin import NVCCPlugin, load_ipython_extension -__version__ = "1.0.0" +__version__ = "1.0.1" From 6150ae57138e1b205ae27dae2e97aaf6da94f94d Mon Sep 17 00:00:00 2001 From: Cosmin Ciocan Date: Fri, 5 Jan 2024 20:46:24 +0100 Subject: [PATCH 23/46] Fix table of contents --- README.md | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/README.md b/README.md index 5e34ef3..290165b 100644 --- a/README.md +++ b/README.md @@ -5,7 +5,6 @@ | Testing | ![Python Versions][python-version] [![CI - Test][test-badge]][test-workflow] [![Coverage][coverage-badge]][coverage-results] | | Package | [![PyPI Latest Release][pypi-latest-version]][pypi-project-url] [![PyPI Downloads][pypi-downloads]][pypi-project-url] | - [python-version]: https://img.shields.io/pypi/pyversions/nvcc4jupyter [test-badge]: https://github.com/cosminc98/nvcc4jupyter/actions/workflows/test.yml/badge.svg [test-workflow]: https://github.com/cosminc98/nvcc4jupyter/actions/workflows/test.yml @@ -15,9 +14,6 @@ [pypi-latest-version]: https://img.shields.io/pypi/v/nvcc4jupyter.svg [pypi-downloads]: https://img.shields.io/pypi/dd/nvcc4jupyter.svg?label=PyPI%20downloads - -## What is it? - **nvcc4jupyter** is a Jupyter Notebook plugin that provides cell and line [magics](https://ipython.readthedocs.io/en/stable/interactive/magics.html) to allow running CUDA C++ code from a notebook. This is especially @@ -30,8 +26,7 @@ to own a GPU yourself. - [Main Features](#main-features) - [Install](#install) -- [Dependencies](#dependencies) -- [Installation from sources](#installation-from-sources) +- [Usage](#usage) - [License](#license) - [Documentation](#documentation) From 1ca949d80389578a6796fd50d695e8f4544dac17 Mon Sep 17 00:00:00 2001 From: Cosmin Ciocan Date: Fri, 12 Jan 2024 14:47:30 +0100 Subject: [PATCH 24/46] Add readthedocs config and sphinx documentation --- .readthedocs.yaml | 32 +++++ docs/Makefile | 20 ++++ docs/make.bat | 35 ++++++ docs/requirements.txt | 2 + docs/source/conf.py | 40 +++++++ docs/source/index.rst | 23 ++++ docs/source/magics.rst | 172 ++++++++++++++++++++++++++ docs/source/usage.rst | 265 +++++++++++++++++++++++++++++++++++++++++ 8 files changed, 589 insertions(+) create mode 100644 .readthedocs.yaml create mode 100644 docs/Makefile create mode 100644 docs/make.bat create mode 100644 docs/requirements.txt create mode 100644 docs/source/conf.py create mode 100644 docs/source/index.rst create mode 100644 docs/source/magics.rst create mode 100644 docs/source/usage.rst diff --git a/.readthedocs.yaml b/.readthedocs.yaml new file mode 100644 index 0000000..7dc180b --- /dev/null +++ b/.readthedocs.yaml @@ -0,0 +1,32 @@ +# .readthedocs.yaml +# Read the Docs configuration file +# See https://docs.readthedocs.io/en/stable/config-file/v2.html for details + +# Required +version: 2 + +# Set the OS, Python version and other tools you might need +build: + os: ubuntu-22.04 + tools: + python: "3.12" + # You can also specify other tool versions: + # nodejs: "19" + # rust: "1.64" + # golang: "1.19" + +# Build documentation in the "docs/" directory with Sphinx +sphinx: + configuration: docs/conf.py + +# Optionally build your docs in additional formats such as PDF and ePub +# formats: +# - pdf +# - epub + +# Optional but recommended, declare the Python requirements required +# to build your documentation +# See https://docs.readthedocs.io/en/stable/guides/reproducible-builds.html +python: + install: + - requirements: docs/requirements.txt \ No newline at end of file diff --git a/docs/Makefile b/docs/Makefile new file mode 100644 index 0000000..269cadc --- /dev/null +++ b/docs/Makefile @@ -0,0 +1,20 @@ +# Minimal makefile for Sphinx documentation +# + +# You can set these variables from the command line, and also +# from the environment for the first two. +SPHINXOPTS ?= +SPHINXBUILD ?= sphinx-build +SOURCEDIR = source +BUILDDIR = build + +# Put it first so that "make" without argument is like "make help". +help: + @$(SPHINXBUILD) -M help "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) + +.PHONY: help Makefile + +# Catch-all target: route all unknown targets to Sphinx using the new +# "make mode" option. $(O) is meant as a shortcut for $(SPHINXOPTS). +%: Makefile + @$(SPHINXBUILD) -M $@ "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) \ No newline at end of file diff --git a/docs/make.bat b/docs/make.bat new file mode 100644 index 0000000..fa98a78 --- /dev/null +++ b/docs/make.bat @@ -0,0 +1,35 @@ +@ECHO OFF + +pushd %~dp0 + +REM Command file for Sphinx documentation + +if "%SPHINXBUILD%" == "" ( + set SPHINXBUILD=sphinx-build +) +set SOURCEDIR=source +set BUILDDIR=build + +if "%1" == "" goto help + +%SPHINXBUILD% >NUL 2>NUL +if errorlevel 9009 ( + echo. + echo.The 'sphinx-build' command was not found. Make sure you have Sphinx + echo.installed, then set the SPHINXBUILD environment variable to point + echo.to the full path of the 'sphinx-build' executable. Alternatively you + echo.may add the Sphinx directory to PATH. + echo. + echo.If you don't have Sphinx installed, grab it from + echo.http://sphinx-doc.org/ + exit /b 1 +) + +%SPHINXBUILD% -M %1 %SOURCEDIR% %BUILDDIR% %SPHINXOPTS% %O% +goto end + +:help +%SPHINXBUILD% -M help %SOURCEDIR% %BUILDDIR% %SPHINXOPTS% %O% + +:end +popd \ No newline at end of file diff --git a/docs/requirements.txt b/docs/requirements.txt new file mode 100644 index 0000000..1d628f2 --- /dev/null +++ b/docs/requirements.txt @@ -0,0 +1,2 @@ +sphinx==7.1.2 +sphinx-rtd-theme==1.3.0rc1 \ No newline at end of file diff --git a/docs/source/conf.py b/docs/source/conf.py new file mode 100644 index 0000000..665059c --- /dev/null +++ b/docs/source/conf.py @@ -0,0 +1,40 @@ +# Configuration file for the Sphinx documentation builder. +# +# For the full list of built-in configuration values, see the documentation: +# https://www.sphinx-doc.org/en/master/usage/configuration.html + +# -- Project information ----------------------------------------------------- +# https://www.sphinx-doc.org/en/master/usage/configuration.html#project-information + +project = "nvcc4jupyter" +copyright = "2024, Andrei Nechaev & Cosmin Stefan Ciocan" +author = "Andrei Nechaev & Cosmin Stefan Ciocan" +release = "1.0.1" +version = "1.0.1" + +# -- General configuration --------------------------------------------------- +# https://www.sphinx-doc.org/en/master/usage/configuration.html#general-configuration + +extensions = [ + "sphinx.ext.duration", + "sphinx.ext.doctest", + "sphinx.ext.autodoc", + "sphinx.ext.autosummary", + "sphinx.ext.intersphinx", +] + +intersphinx_mapping = { + "python": ("https://docs.python.org/3/", None), + "sphinx": ("https://www.sphinx-doc.org/en/master/", None), +} +intersphinx_disabled_domains = ["std"] + +templates_path = ["_templates"] +exclude_patterns = [] + + +# -- Options for HTML output ------------------------------------------------- +# https://www.sphinx-doc.org/en/master/usage/configuration.html#options-for-html-output + +html_theme = "sphinx_rtd_theme" +html_static_path = ["_static"] diff --git a/docs/source/index.rst b/docs/source/index.rst new file mode 100644 index 0000000..b472c1d --- /dev/null +++ b/docs/source/index.rst @@ -0,0 +1,23 @@ +Welcome to nvcc4jupyter's documentation! +======================================== + +.. note:: + + This project is under active development. + +Contents +-------- + +.. toctree:: + :maxdepth: 2 + :caption: Contents: + + usage + magics + +Indices and tables +================== + +* :ref:`genindex` +* :ref:`modindex` +* :ref:`search` diff --git a/docs/source/magics.rst b/docs/source/magics.rst new file mode 100644 index 0000000..2073f35 --- /dev/null +++ b/docs/source/magics.rst @@ -0,0 +1,172 @@ +********** +Magics API +********** + +.. note:: + Arguments for profilers and the nvcc compiler can be passed in double + quotes so they can contain spaces and dashes. + +------ + +.. _cuda_magic: + +cuda +==== + +Magic command that compiles, runs, and profiles CUDA C++ code in the cell. + +Usage +----- + + - ``%%cuda``: Compile and run this cell. + - ``%%cuda -p``: Also runs the Nsight Compute profiler. + - ``%%cuda -p -a ""``: Also runs the Nsight Compute profiler. + - ``%%cuda -t``: Outputs the "timeit" built-in magic results. + +Options +------- + +-t, --timeit + Boolean. If set, returns the output of the "timeit" built-in + ipython magic instead of stdout. + +-p, --profile + Boolean. If set, runs the NVIDIA Nsight Compute profiler whose + output is appended to standard output. + +-a, --profiler-args + String. Optional profiler arguments that can be space separated + by wrapping them in double quotes. See all options here: + `Nsight Compute CLI `_ + +.. note:: + If both "\-\-profile" and "\-\-timeit" are used then no profiling is + done. + +Examples +-------- +:: + + # compile, run, and profile the code in the cell with the Nsight + # compute profiler while collecting only metrics from the + # "MemoryWorkloadAnalysis" section. + %%cuda --profile --profiler-args "--section MemoryWorkloadAnalysis" + +------ + +.. _cuda_group_save_magic: + +cuda_group_save +=============== + +Magic command that saves CUDA C++ code in the cell for later +compilation and execution with possibly more source files. + +Usage +----- + + - ``%%cuda_group_save -n -g ``: Save the code in the current cell to a group of source files. + +Options +------- + +-n, --name + String. Required file name of the saved source file. Must have + either the ".cu" or ".h" extension. In order to import a header + file saved with this magic you can simply add '#include ""'. + +-g, --group + String. Required group name to which to add the saved source file. + Groups are source files that get compiled together and do not + interact with other groups. This allows you to have multiple + unrelated CUDA programs within the same jupyter notebook. Adding + files to a group named "shared" will make them available to all + other source file groups. One use case for the shared group is for + sharing error handling code which should be present in all CUDA + programs. + +Examples +-------- +:: + + # jupyter cell 1 + %%cuda_group_save -n "error_handling.h" -g "shared" + + + # jupyter cell 2 + %%cuda_group_save -n "main.cu" -g "example_group" + #include "error_handling.h" + + +------ + +.. _cuda_group_run_magic: + +cuda_group_run +============== + +Line magic command that compiles, runs, and profiles all source files +in a group. + +Usage +----- + + - ``%%cuda_group_run -g ``: Compiles, runs, and profiles the sources files in the given group. + +Options +------- + +-g, --group + String. Required group name whose source files should be deleted. + +.. note:: + All options from the "%%cuda" cell magic are inherited. + +Examples +-------- +:: + + # jupyter cell 1 + %%cuda_group_save -n "error_handling.h" -g "shared" + + + # jupyter cell 2 + %%cuda_group_save -n "main.cu" -g "example_group" + #include "error_handling.h" + + + # jupyter cell 3 + %cuda_group_run -g "example_group" --profile + +----- + +.. _cuda_group_delete_magic: + +cuda_group_delete +================= + +Line magic command that deletes all source files in a group. + +Usage +----- + + - ``%%cuda_group_delete -g ``: Removes all source files in the given group. + +Options +------- + +-g, --group + String. Required group name whose source files should be deleted. + +Examples +-------- +:: + + # jupyter cell 1 + %%cuda_group_save -n "error_handling.h" -g "shared" + + + # jupyter cell 2 - here we delete the error shared group; in + # practice this would be helpful if you want to overwrite some + # functionality that was defined earlier in the notebook + %cuda_group_delete -g "shared" diff --git a/docs/source/usage.rst b/docs/source/usage.rst new file mode 100644 index 0000000..885e419 --- /dev/null +++ b/docs/source/usage.rst @@ -0,0 +1,265 @@ +Usage +===== + +This IPython extension allows running CUDA C++ code in Jupyter notebook. This +is especially useful when combined with `Google Colab `_ +which provides CUDA capable GPUs with the CUDA toolkit already installed. + +.. _installation: + +Installation +------------ + +To use nvcc4jupyter, first install it using pip: + +.. code-block:: console + + (venv) $ pip install nvcc4jupyter + +.. _load_extension: + +Load the Extension +------------------ + +Now we need to load the IPython extension to be able to use its cell and line +magic commands: + +.. code-block:: + + %load_ext nvcc4jupyter + +Hello World +----------- + +We will use the :ref:`cuda ` cell magic command to run a simple +hello world program. + +.. code-block:: c++ + + %%cuda + #include + + __global__ void hello(){ + printf("Hello from block: %u, thread: %u\n", blockIdx.x, threadIdx.x); + } + + int main(){ + hello<<<2, 2>>>(); + cudaDeviceSynchronize(); + } + +Groups +------ + +Now we will demonstrate a more complex scenario that uses source file groups. +If you want to split your code into multiple source files, either for code reuse +or just to have an easier to read project, you want to use groups. A group of +source files will be compiled together. Because of this, you can include headers +from the same group and use the code defined in other ".cu" files. There is also +a special group named "shared" whose files will be compiled together with all +other groups, which is a great feature for error handling code as we'll show now: + +.. code-block:: c++ + + %%cuda_group_save --group shared --name "error_handling.h" + // error checking macro + #define cudaCheckErrors(msg) \ + do { \ + cudaError_t __err = cudaGetLastError(); \ + if (__err != cudaSuccess) { \ + fprintf(stderr, "Fatal error: %s (%s at %s:%d)\n", \ + msg, cudaGetErrorString(__err), \ + __FILE__, __LINE__); \ + fprintf(stderr, "*** FAILED - ABORTING\n"); \ + exit(1); \ + } \ + } while (0) + +Now we can use that error handling macro in this vector addition program but +also in other programs that we define in other Jupyter cells: + +.. code-block:: c++ + + %%cuda + #include + #include "error_handling.h" + + const int DSIZE = 4096; + const int block_size = 256; + + // vector add kernel: C = A + B + __global__ void vadd(const float *A, const float *B, float *C, int ds){ + int idx = threadIdx.x + blockIdx.x * blockDim.x; + if (idx < ds) { + C[idx] = A[idx] + B[idx]; + } + } + + int main(){ + float *h_A, *h_B, *h_C, *d_A, *d_B, *d_C; + + // allocate space for vectors in host memory + h_A = new float[DSIZE]; + h_B = new float[DSIZE]; + h_C = new float[DSIZE]; + + // initialize vectors in host memory to random values (except for the + // result vector whose values do not matter as they will be overwritten) + for (int i = 0; i < DSIZE; i++) { + h_A[i] = rand()/(float)RAND_MAX; + h_B[i] = rand()/(float)RAND_MAX; + } + + // allocate space for vectors in device memory + cudaMalloc(&d_A, DSIZE*sizeof(float)); + cudaMalloc(&d_B, DSIZE*sizeof(float)); + cudaMalloc(&d_C, DSIZE*sizeof(float)); + cudaCheckErrors("cudaMalloc failure"); // error checking + + // copy vectors A and B from host to device: + cudaMemcpy(d_A, h_A, DSIZE*sizeof(float), cudaMemcpyHostToDevice); + cudaMemcpy(d_B, h_B, DSIZE*sizeof(float), cudaMemcpyHostToDevice); + cudaCheckErrors("cudaMemcpy H2D failure"); + + // launch the vector adding kernel + vadd<<<(DSIZE+block_size-1)/block_size, block_size>>>(d_A, d_B, d_C, DSIZE); + cudaCheckErrors("kernel launch failure"); + + // wait for the kernel to finish execution + cudaDeviceSynchronize(); + cudaCheckErrors("kernel execution failure"); + + cudaMemcpy(h_C, d_C, DSIZE*sizeof(float), cudaMemcpyDeviceToHost); + cudaCheckErrors("cudaMemcpy D2H failure"); + + printf("A[0] = %f\n", h_A[0]); + printf("B[0] = %f\n", h_B[0]); + printf("C[0] = %f\n", h_C[0]); + return 0; + } + +Above we use the :ref:`cuda ` magic command which saves the code +in the cell to an anonymous source file group, compiles, and executes that +code. This only allows us to have one source file (besides the ones in the +"shared" group). In order to have multiple source files we need to use the +:ref:`cuda_group_save ` and +:ref:`cuda_group_run ` magics. + +First, we save the vector addition function to its own file: + + +.. code-block:: c++ + + %%cuda_group_save --name "vector_add.cu" --group "vector_add" + // vector add kernel: C = A + B + __global__ void vadd(const float *A, const float *B, float *C, int ds){ + int idx = threadIdx.x + blockIdx.x * blockDim.x; + if (idx < ds) { + C[idx] = A[idx] + B[idx]; + } + } + +Now we create a header file so the main cuda file knows the signature of "vadd": + +.. code-block:: c++ + + %%cuda_group_save --name "vector_add.h" --group "vector_add" + __global__ void vadd(const float *A, const float *B, float *C, int ds); + +To tie it all together, we save the main cuda file, which includes our vector +addition code: + +.. code-block:: c++ + + %%cuda_group_save --name "main.cu" --group "vector_add" + #include + #include "error_handling.h" + #include "vector_add.h" + + const int DSIZE = 4096; + const int block_size = 256; + + int main(){ + float *h_A, *h_B, *h_C, *d_A, *d_B, *d_C; + + // allocate space for vectors in host memory + h_A = new float[DSIZE]; + h_B = new float[DSIZE]; + h_C = new float[DSIZE]; + + // initialize vectors in host memory to random values (except for the + // result vector whose values do not matter as they will be overwritten) + for (int i = 0; i < DSIZE; i++) { + h_A[i] = rand()/(float)RAND_MAX; + h_B[i] = rand()/(float)RAND_MAX; + } + + // allocate space for vectors in device memory + cudaMalloc(&d_A, DSIZE*sizeof(float)); + cudaMalloc(&d_B, DSIZE*sizeof(float)); + cudaMalloc(&d_C, DSIZE*sizeof(float)); + cudaCheckErrors("cudaMalloc failure"); // error checking + + // copy vectors A and B from host to device: + cudaMemcpy(d_A, h_A, DSIZE*sizeof(float), cudaMemcpyHostToDevice); + cudaMemcpy(d_B, h_B, DSIZE*sizeof(float), cudaMemcpyHostToDevice); + cudaCheckErrors("cudaMemcpy H2D failure"); + + // launch the vector adding kernel + vadd<<<(DSIZE+block_size-1)/block_size, block_size>>>(d_A, d_B, d_C, DSIZE); + cudaCheckErrors("kernel launch failure"); + + // wait for the kernel to finish execution + cudaDeviceSynchronize(); + cudaCheckErrors("kernel execution failure"); + + cudaMemcpy(h_C, d_C, DSIZE*sizeof(float), cudaMemcpyDeviceToHost); + cudaCheckErrors("cudaMemcpy D2H failure"); + + printf("A[0] = %f\n", h_A[0]); + printf("B[0] = %f\n", h_B[0]); + printf("C[0] = %f\n", h_C[0]); + return 0; + } + +Now we can compile all the source files in the group and execute the main +function with the following command: + +.. code-block:: c++ + + %cuda_group_run --group "vector_add" + +Profiling +--------- + +Another important feature of nvcc4jupyter is its integration with the NVIDIA +Nsight Compute profiler, which you need to make sure is installed and its +executable can be found in a directory in your PATH environment variable. + +In order to use it and provide the profiler with custom arguments, simply run: + +.. code-block:: c++ + + %cuda_group_run --group "vector_add" --profile --profiler-args "--section SpeedOfLight" + +Running the cell above will compile and execute the vector addition code in the +"vector_add" group and profile it, keeping only the metrics from the +"SpeedOfLight" section. The output will contain something similar to: + +.. code-block:: + + Section: GPU Speed Of Light Throughput + ----------------------- ------------- ------------ + Metric Name Metric Unit Metric Value + ----------------------- ------------- ------------ + DRAM Frequency cycle/nsecond 4.65 + SM Frequency cycle/usecond 544.31 + Elapsed Cycles cycle 2,145 + Memory Throughput % 3.19 + DRAM Throughput % 3.19 + Duration usecond 3.94 + L1/TEX Cache Throughput % 6.67 + L2 Cache Throughput % 1.98 + SM Active Cycles cycle 383.65 + Compute (SM) Throughput % 1.19 + ----------------------- ------------- ------------ From 5bfd81a5531bba2340ec047cc126e11ddb972231 Mon Sep 17 00:00:00 2001 From: Cosmin Ciocan Date: Fri, 12 Jan 2024 15:14:44 +0100 Subject: [PATCH 25/46] Change python 3.12 to 3.10 in readthedocs config --- .readthedocs.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.readthedocs.yaml b/.readthedocs.yaml index 7dc180b..1315873 100644 --- a/.readthedocs.yaml +++ b/.readthedocs.yaml @@ -9,7 +9,7 @@ version: 2 build: os: ubuntu-22.04 tools: - python: "3.12" + python: "3.10" # You can also specify other tool versions: # nodejs: "19" # rust: "1.64" From 55423ce07ae8fb365fe8d9aca0e3e6ca333dae9f Mon Sep 17 00:00:00 2001 From: Cosmin Ciocan Date: Fri, 12 Jan 2024 15:17:34 +0100 Subject: [PATCH 26/46] Update conf.py path in readthedocs config --- .readthedocs.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.readthedocs.yaml b/.readthedocs.yaml index 1315873..eb12299 100644 --- a/.readthedocs.yaml +++ b/.readthedocs.yaml @@ -17,7 +17,7 @@ build: # Build documentation in the "docs/" directory with Sphinx sphinx: - configuration: docs/conf.py + configuration: docs/source/conf.py # Optionally build your docs in additional formats such as PDF and ePub # formats: From 3cf0d60d1ae9702a261f5c0c3ecadfb52eb99a98 Mon Sep 17 00:00:00 2001 From: Cosmin Ciocan Date: Fri, 12 Jan 2024 15:27:05 +0100 Subject: [PATCH 27/46] Move project description from usage page to home page --- docs/source/index.rst | 16 +++------------- docs/source/usage.rst | 8 -------- 2 files changed, 3 insertions(+), 21 deletions(-) diff --git a/docs/source/index.rst b/docs/source/index.rst index b472c1d..1f07bdd 100644 --- a/docs/source/index.rst +++ b/docs/source/index.rst @@ -1,12 +1,9 @@ Welcome to nvcc4jupyter's documentation! ======================================== -.. note:: - - This project is under active development. - -Contents --------- +This IPython extension allows running CUDA C++ code in Jupyter notebook. This +is especially useful when combined with `Google Colab `_ +which provides CUDA capable GPUs with the CUDA toolkit already installed. .. toctree:: :maxdepth: 2 @@ -14,10 +11,3 @@ Contents usage magics - -Indices and tables -================== - -* :ref:`genindex` -* :ref:`modindex` -* :ref:`search` diff --git a/docs/source/usage.rst b/docs/source/usage.rst index 885e419..38ff35c 100644 --- a/docs/source/usage.rst +++ b/docs/source/usage.rst @@ -1,12 +1,6 @@ Usage ===== -This IPython extension allows running CUDA C++ code in Jupyter notebook. This -is especially useful when combined with `Google Colab `_ -which provides CUDA capable GPUs with the CUDA toolkit already installed. - -.. _installation: - Installation ------------ @@ -16,8 +10,6 @@ To use nvcc4jupyter, first install it using pip: (venv) $ pip install nvcc4jupyter -.. _load_extension: - Load the Extension ------------------ From ad020e1231f5ac903b771c6934a6bb3e429b8294 Mon Sep 17 00:00:00 2001 From: Cosmin Ciocan Date: Fri, 12 Jan 2024 15:40:48 +0100 Subject: [PATCH 28/46] Add MIT license --- LICENSE | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) create mode 100644 LICENSE diff --git a/LICENSE b/LICENSE new file mode 100644 index 0000000..47b69e9 --- /dev/null +++ b/LICENSE @@ -0,0 +1,22 @@ +MIT License + +Copyright (c) 2018-2024 Andrei Nechaev, Cosmin Stefan Ciocan and others + +Permission is hereby granted, free of charge, to any person obtaining +a copy of this software and associated documentation files (the +"Software"), to deal in the Software without restriction, including +without limitation the rights to use, copy, modify, merge, publish, +distribute, sublicense, and/or sell copies of the Software, and to +permit persons to whom the Software is furnished to do so, subject to +the following conditions: + +The above copyright notice and this permission notice shall be +included in all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE +LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION +OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION +WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. \ No newline at end of file From 4d059df78d2d7a3638cd4c286d2b023306c1c407 Mon Sep 17 00:00:00 2001 From: Cosmin Ciocan Date: Fri, 12 Jan 2024 15:41:17 +0100 Subject: [PATCH 29/46] Update metadata in pyproject.toml --- pyproject.toml | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index 3a99a3a..0b46a03 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -6,10 +6,11 @@ build-backend = "hatchling.build" name = "nvcc4jupyter" description = "Jupyter notebook plugin to run CUDA C/C++ code" readme = "README.md" -requires-python = ">=3.7" +requires-python = ">=3.10" license = {text = "MIT License"} authors = [ { name = "Andrei Nechaev", email = "lyfaradey@yahoo.com" }, + { name = "Cosmin Stefan Ciocan", email = "ciocan.cosmin98@gmail.com }, ] classifiers = [ "Programming Language :: Python", @@ -17,10 +18,18 @@ classifiers = [ 'Programming Language :: Python :: 3.10', 'Programming Language :: Python :: 3.11', 'Programming Language :: Python :: 3.12', + 'Environment :: GPU', + 'Environment :: GPU :: NVIDIA CUDA', + 'Framework :: IPython', + 'Framework :: Jupyter', ] dependencies = [] dynamic = ["version"] +[project.urls] +documentation = 'https://nvcc4jupyter.readthedocs.io/' +repository = 'https://github.com/andreinechaev/nvcc4jupyter' + [tool.hatch.version] path = "nvcc4jupyter/__init__.py" From 01bef596ca2e69d0abe2d37a400b71c7e7736193 Mon Sep 17 00:00:00 2001 From: Cosmin Ciocan Date: Fri, 12 Jan 2024 15:41:58 +0100 Subject: [PATCH 30/46] Change daily downloads to monthly downloads in README.md badge --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 290165b..3d52f67 100644 --- a/README.md +++ b/README.md @@ -12,7 +12,7 @@ [coverage-results]: https://codecov.io/gh/cosminc98/nvcc4jupyter [pypi-project-url]: https://pypi.org/project/nvcc4jupyter/ [pypi-latest-version]: https://img.shields.io/pypi/v/nvcc4jupyter.svg -[pypi-downloads]: https://img.shields.io/pypi/dd/nvcc4jupyter.svg?label=PyPI%20downloads +[pypi-downloads]: https://img.shields.io/pypi/dm/nvcc4jupyter.svg?label=PyPI%20downloads **nvcc4jupyter** is a Jupyter Notebook plugin that provides cell and line [magics](https://ipython.readthedocs.io/en/stable/interactive/magics.html) From 0714d3af192f2387e631f4ac8846742c412ae461 Mon Sep 17 00:00:00 2001 From: Cosmin Ciocan Date: Fri, 12 Jan 2024 15:44:15 +0100 Subject: [PATCH 31/46] Fix missing double quote --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index 0b46a03..b658c20 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -10,7 +10,7 @@ requires-python = ">=3.10" license = {text = "MIT License"} authors = [ { name = "Andrei Nechaev", email = "lyfaradey@yahoo.com" }, - { name = "Cosmin Stefan Ciocan", email = "ciocan.cosmin98@gmail.com }, + { name = "Cosmin Stefan Ciocan", email = "ciocan.cosmin98@gmail.com" }, ] classifiers = [ "Programming Language :: Python", From dff5b2753e742992b1f68c3e3ad3bfb9dde174bf Mon Sep 17 00:00:00 2001 From: Cosmin Ciocan Date: Fri, 12 Jan 2024 15:53:17 +0100 Subject: [PATCH 32/46] Remove help from parser arguments and instead provide links in the parser description to documentation to avoid duplicate text --- nvcc4jupyter/parsers.py | 91 ++++++++--------------------------------- 1 file changed, 18 insertions(+), 73 deletions(-) diff --git a/nvcc4jupyter/parsers.py b/nvcc4jupyter/parsers.py index 62a1c15..201aaf5 100644 --- a/nvcc4jupyter/parsers.py +++ b/nvcc4jupyter/parsers.py @@ -5,53 +5,25 @@ def get_parser_cuda() -> argparse.ArgumentParser: parser = argparse.ArgumentParser( description=( "%%cuda magic that compiles and runs CUDA C++ code in this cell." + " See https://nvcc4jupyter.readthedocs.io/en/latest/magics.html#cuda" + " for usage details." ) ) - parser.add_argument( - "-t", - "--timeit", - action="store_true", - help=( - 'If set, returns the output of the "timeit" built-in ipython magic' - " instead of stdout." - ), - ) - parser.add_argument( - "-p", - "--profile", - action="store_true", - help=( - "If set, runs the nvidia nsight compute profiler. Has no effect if" - " used with --timeit." - ), - ) - parser.add_argument( - "-a", - "--profiler-args", - type=str, - default="", - help=( - "Extra options that can be passed to the nvidia nsight compute" - " profiler. Must be the last option given to the argument parser" - " so you can pass arguments with dashes." - ), - ) + parser.add_argument("-t", "--timeit", action="store_true") + parser.add_argument("-p", "--profile", action="store_true") + parser.add_argument("-a", "--profiler-args", type=str, default="") return parser def get_parser_cuda_group_run() -> argparse.ArgumentParser: parser = get_parser_cuda() parser.description = ( - "%%cuda_group_run magic that compiles and runs source files in a" - " given group." - ) - parser.add_argument( - "-g", - "--group", - type=str, - required=True, - help="The group whose files should be compiled and executed.", + "%%cuda_group_run magic that compiles and runs source files in a given" + " group. See" + " https://nvcc4jupyter.readthedocs.io/en/latest/magics.html#cuda-group-run" + " for usage details." ) + parser.add_argument("-g", "--group", type=str, required=True) return parser @@ -60,49 +32,22 @@ def get_parser_cuda_group_save() -> argparse.ArgumentParser: description=( "%%cuda_group_save magic that saves CUDA C++ code in this cell for" " later compilation and execution with possibly more source files." + " See https://nvcc4jupyter.readthedocs.io/en/latest/magics.html#cuda-group-save" + " for usage details." ) ) - parser.add_argument( - "-n", - "--name", - type=str, - required=True, - help=( - 'The name of the saved source file. Must have either the ".cu" or' - ' ".h" extension. In order to import a header file saved with this' - " magic you can simply add '#include \"\"'." - ), - ) - parser.add_argument( - "-g", - "--group", - type=str, - required=True, - help=( - "The group to which to add the saved source file. Groups are" - " source files that get compiled together and do not interact with" - " other groups. This allows you to have multiple unrelated CUDA" - " programs within the same jupyter notebook. Adding files to a" - ' group named "shared" will make them available to all other' - " source file groups. One use case for this is sharing error" - " handling code." - ), - ) + parser.add_argument("-n", "--name", type=str, required=True) + parser.add_argument("-g", "--group", type=str, required=True) return parser def get_parser_cuda_group_delete() -> argparse.ArgumentParser: parser = argparse.ArgumentParser( description=( - "%%cuda_group_reset magic that deletes all files in a group." + "%%cuda_group_delete magic that deletes all files in a group. See" + " https://nvcc4jupyter.readthedocs.io/en/latest/magics.html#cuda-group-delete" + " for usage details." ) ) - parser.add_argument( - "-g", - "--group", - type=str, - required=True, - help="The group whose files should be deleted.", - ) + parser.add_argument("-g", "--group", type=str, required=True) return parser - From ece222d3db7288011aa67dcbcb47f4e6a4fd7269 Mon Sep 17 00:00:00 2001 From: Cosmin Ciocan Date: Fri, 12 Jan 2024 16:04:39 +0100 Subject: [PATCH 33/46] Update README.md with usage and documentation links --- README.md | 32 +++++++++++++++++++++++++++----- 1 file changed, 27 insertions(+), 5 deletions(-) diff --git a/README.md b/README.md index 3d52f67..d7ff5b4 100644 --- a/README.md +++ b/README.md @@ -33,8 +33,9 @@ to own a GPU yourself. ## Main Features Here are just a few of the things that nvcc4jupyter does well: - - TODO1 - - TODO2 + - [Easily run CUDA C++ code](https://nvcc4jupyter.readthedocs.io/en/latest/usage.html#hello-world) + - [Profile your code with NVIDIA Nsight Compute](https://nvcc4jupyter.readthedocs.io/en/latest/usage.html#profiling) + - [Share code between different programs in the same notebook / split your code into multiple files for improved readability](https://nvcc4jupyter.readthedocs.io/en/latest/usage.html#groups) ## Install The installer for the latest released version is available at the [Python @@ -45,13 +46,34 @@ pip install nvcc4jupyter ``` ## Usage -TODO + +First, load the extension to enable the magic commands: +``` +%load_ext nvcc4jupyter +``` + +Running a quick CUDA Hello World program: +```c++ +%%cuda +#include + +__global__ void hello(){ + printf("Hello from block: %u, thread: %u\n", blockIdx.x, threadIdx.x); +} + +int main(){ + hello<<<2, 2>>>(); + cudaDeviceSynchronize(); +} +``` + +For more advanced use cases, see [the documentation](https://nvcc4jupyter.readthedocs.io/en/latest/usage.html). ## Documentation -The official documentation is hosted on [TODO](TODO). +The official documentation is hosted on [readthedocs](https://nvcc4jupyter.readthedocs.io/). ## License -[TODO](LICENSE) +[MIT](LICENSE)
From 6352f21c04291af369649f171d7dd443385198b9 Mon Sep 17 00:00:00 2001 From: Cosmin Ciocan Date: Fri, 12 Jan 2024 16:05:38 +0100 Subject: [PATCH 34/46] Update version to 1.0.2 --- nvcc4jupyter/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nvcc4jupyter/__init__.py b/nvcc4jupyter/__init__.py index 08a8aff..33266ea 100644 --- a/nvcc4jupyter/__init__.py +++ b/nvcc4jupyter/__init__.py @@ -1,3 +1,3 @@ from .plugin import NVCCPlugin, load_ipython_extension -__version__ = "1.0.1" +__version__ = "1.0.2" From c38fb5421663553ac7dd155b718f28fbab49d750 Mon Sep 17 00:00:00 2001 From: Cosmin Ciocan Date: Fri, 12 Jan 2024 16:41:41 +0100 Subject: [PATCH 35/46] Add pre-commit hook with black config --- .pre-commit-config.yaml | 25 +++++++++++++++++++++++++ 1 file changed, 25 insertions(+) create mode 100644 .pre-commit-config.yaml diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml new file mode 100644 index 0000000..0cd98a8 --- /dev/null +++ b/.pre-commit-config.yaml @@ -0,0 +1,25 @@ +default_language_version: + python: python3 + +repos: + - repo: https://github.com/pre-commit/pre-commit-hooks + rev: v4.4.0 + hooks: + # list of supported hooks: https://pre-commit.com/hooks.html + - id: trailing-whitespace + - id: end-of-file-fixer + - id: check-docstring-first + - id: check-yaml + - id: debug-statements + - id: detect-private-key + - id: check-executables-have-shebangs + - id: check-toml + - id: check-case-conflict + - id: check-added-large-files + + # python code formatting + - repo: https://github.com/psf/black + rev: 23.1.0 + hooks: + - id: black + args: [--line-length, "79", --experimental-string-processing] From 2eae513950f220aa7c3b2191fee764dff661171d Mon Sep 17 00:00:00 2001 From: Cosmin Ciocan Date: Fri, 12 Jan 2024 16:50:26 +0100 Subject: [PATCH 36/46] Add dev optional dependency --- pyproject.toml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index b658c20..2e5ca54 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -38,6 +38,7 @@ packages = ["nvcc4jupyter"] [project.optional-dependencies] testing = ["pytest>=7.4.3", "IPython>=8.19.0"] +dev = ["pytest>=7.4.3", "IPython>=8.19.0", "pre-commit>=3.6.0", "pytest-cov[toml]>=4.1.0"] [tool.pytest.ini_options] @@ -306,4 +307,4 @@ deprecated-modules="optparse,tkinter.tix" overgeneral-exceptions= [ "BaseException", "Exception" -] \ No newline at end of file +] From ef642e3480d834ce55a80a751efda803031f9cef Mon Sep 17 00:00:00 2001 From: Cosmin Ciocan Date: Fri, 12 Jan 2024 16:51:51 +0100 Subject: [PATCH 37/46] Add newlines at the end of the files from pre-commit hook --- .readthedocs.yaml | 2 +- .vscode/settings.json | 2 +- LICENSE | 2 +- docs/Makefile | 2 +- docs/make.bat | 2 +- docs/requirements.txt | 2 +- tests/fixtures/multiple_files/hello.cu | 2 +- tests/fixtures/multiple_files/hello.h | 2 +- tests/fixtures/multiple_files/main.cu | 2 +- tests/fixtures/single_file/hello.cu | 2 +- tests/requirements.txt | 2 +- 11 files changed, 11 insertions(+), 11 deletions(-) diff --git a/.readthedocs.yaml b/.readthedocs.yaml index eb12299..314290f 100644 --- a/.readthedocs.yaml +++ b/.readthedocs.yaml @@ -29,4 +29,4 @@ sphinx: # See https://docs.readthedocs.io/en/stable/guides/reproducible-builds.html python: install: - - requirements: docs/requirements.txt \ No newline at end of file + - requirements: docs/requirements.txt diff --git a/.vscode/settings.json b/.vscode/settings.json index 0e16f80..6422f59 100644 --- a/.vscode/settings.json +++ b/.vscode/settings.json @@ -29,4 +29,4 @@ "isort.args": [ "--settings-path=pyproject.toml" ] -} \ No newline at end of file +} diff --git a/LICENSE b/LICENSE index 47b69e9..134312e 100644 --- a/LICENSE +++ b/LICENSE @@ -19,4 +19,4 @@ MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION -WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. \ No newline at end of file +WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. diff --git a/docs/Makefile b/docs/Makefile index 269cadc..d0c3cbf 100644 --- a/docs/Makefile +++ b/docs/Makefile @@ -17,4 +17,4 @@ help: # Catch-all target: route all unknown targets to Sphinx using the new # "make mode" option. $(O) is meant as a shortcut for $(SPHINXOPTS). %: Makefile - @$(SPHINXBUILD) -M $@ "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) \ No newline at end of file + @$(SPHINXBUILD) -M $@ "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) diff --git a/docs/make.bat b/docs/make.bat index fa98a78..319c288 100644 --- a/docs/make.bat +++ b/docs/make.bat @@ -32,4 +32,4 @@ goto end %SPHINXBUILD% -M help %SOURCEDIR% %BUILDDIR% %SPHINXOPTS% %O% :end -popd \ No newline at end of file +popd diff --git a/docs/requirements.txt b/docs/requirements.txt index 1d628f2..53fc1f3 100644 --- a/docs/requirements.txt +++ b/docs/requirements.txt @@ -1,2 +1,2 @@ sphinx==7.1.2 -sphinx-rtd-theme==1.3.0rc1 \ No newline at end of file +sphinx-rtd-theme==1.3.0rc1 diff --git a/tests/fixtures/multiple_files/hello.cu b/tests/fixtures/multiple_files/hello.cu index 3b7a67f..7f6c3c6 100644 --- a/tests/fixtures/multiple_files/hello.cu +++ b/tests/fixtures/multiple_files/hello.cu @@ -3,4 +3,4 @@ __host__ void hello(){ printf("Hello World!\n"); -} \ No newline at end of file +} diff --git a/tests/fixtures/multiple_files/hello.h b/tests/fixtures/multiple_files/hello.h index aaa8cdd..f19e5d3 100644 --- a/tests/fixtures/multiple_files/hello.h +++ b/tests/fixtures/multiple_files/hello.h @@ -3,4 +3,4 @@ void hello(); -#endif \ No newline at end of file +#endif diff --git a/tests/fixtures/multiple_files/main.cu b/tests/fixtures/multiple_files/main.cu index 290472a..5c0ebb8 100644 --- a/tests/fixtures/multiple_files/main.cu +++ b/tests/fixtures/multiple_files/main.cu @@ -3,4 +3,4 @@ int main() { hello(); return 0; -} \ No newline at end of file +} diff --git a/tests/fixtures/single_file/hello.cu b/tests/fixtures/single_file/hello.cu index 5620101..eda41aa 100644 --- a/tests/fixtures/single_file/hello.cu +++ b/tests/fixtures/single_file/hello.cu @@ -7,4 +7,4 @@ __host__ void hello(){ int main() { hello(); return 0; -} \ No newline at end of file +} diff --git a/tests/requirements.txt b/tests/requirements.txt index 78d9bb7..aac9161 100644 --- a/tests/requirements.txt +++ b/tests/requirements.txt @@ -1,2 +1,2 @@ pytest>=7.4.3 -IPython>=8.19.0 \ No newline at end of file +IPython>=8.19.0 From ee68b4025bf5c1e4639953bd3f12f64534c52104 Mon Sep 17 00:00:00 2001 From: Cosmin Ciocan Date: Fri, 12 Jan 2024 16:56:49 +0100 Subject: [PATCH 38/46] Add README.md instructions for developers --- README.md | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/README.md b/README.md index d7ff5b4..f82f00f 100644 --- a/README.md +++ b/README.md @@ -29,6 +29,7 @@ to own a GPU yourself. - [Usage](#usage) - [License](#license) - [Documentation](#documentation) +- [Contributing](#contributing) ## Main Features Here are just a few of the things that nvcc4jupyter does well: @@ -75,6 +76,18 @@ The official documentation is hosted on [readthedocs](https://nvcc4jupyter.readt ## License [MIT](LICENSE) +## Contributing + +Install the package with the development dependencies: +```bash +pip install .[dev] +``` + +As a developer, make sure you install the pre-commit hook before commiting any changes: +```bash +pre-commit install +``` +
[Go to Top](#table-of-contents) From 182e3519ad5148f0f1f17f305192c5164753b4aa Mon Sep 17 00:00:00 2001 From: Cosmin Ciocan Date: Fri, 12 Jan 2024 17:45:37 +0100 Subject: [PATCH 39/46] Move flake8 config from toml to .flake8 as flake8 does not play nice with pyproject.toml and add pre-commit hook for flake8 --- .flake8 | 5 +++++ .pre-commit-config.yaml | 16 +++++++++++++++- .vscode/settings.json | 3 ++- nvcc4jupyter/__init__.py | 2 +- nvcc4jupyter/parsers.py | 8 ++++---- nvcc4jupyter/plugin.py | 4 ++-- pyproject.toml | 22 ---------------------- tests/conftest.py | 2 +- tests/fixtures/fixtures.py | 2 +- 9 files changed, 31 insertions(+), 33 deletions(-) create mode 100644 .flake8 diff --git a/.flake8 b/.flake8 new file mode 100644 index 0000000..bd44966 --- /dev/null +++ b/.flake8 @@ -0,0 +1,5 @@ +[flake8] +max-line-length = 79 +select = F,E,W,B,B901,B902,B903 +exclude = .eggs,.git,.tox,nssm,obj,out,packages,pywin32,tests,swagger_client +ignore = E722,B001,W503,E203 diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 0cd98a8..46ee56e 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -22,4 +22,18 @@ repos: rev: 23.1.0 hooks: - id: black - args: [--line-length, "79", --experimental-string-processing] + args: ["--config", "pyproject.toml"] + + # python import sorting + - repo: https://github.com/PyCQA/isort + rev: 5.12.0 + hooks: + - id: isort + args: ["--settings-path", "pyproject.toml"] + + # python check (PEP8), programming errors and code complexity + - repo: https://github.com/PyCQA/flake8 + rev: 6.0.0 + hooks: + - id: flake8 + args: ["--config", ".flake8"] diff --git a/.vscode/settings.json b/.vscode/settings.json index 6422f59..4785bc9 100644 --- a/.vscode/settings.json +++ b/.vscode/settings.json @@ -24,7 +24,8 @@ "--config=pyproject.toml" ], "flake8.args": [ - "--toml-config=pyproject.toml" + "--config", + ".flake8" ], "isort.args": [ "--settings-path=pyproject.toml" diff --git a/nvcc4jupyter/__init__.py b/nvcc4jupyter/__init__.py index 33266ea..e9f7145 100644 --- a/nvcc4jupyter/__init__.py +++ b/nvcc4jupyter/__init__.py @@ -1,3 +1,3 @@ -from .plugin import NVCCPlugin, load_ipython_extension +from .plugin import NVCCPlugin, load_ipython_extension # noqa: F401 __version__ = "1.0.2" diff --git a/nvcc4jupyter/parsers.py b/nvcc4jupyter/parsers.py index 201aaf5..7679465 100644 --- a/nvcc4jupyter/parsers.py +++ b/nvcc4jupyter/parsers.py @@ -5,7 +5,7 @@ def get_parser_cuda() -> argparse.ArgumentParser: parser = argparse.ArgumentParser( description=( "%%cuda magic that compiles and runs CUDA C++ code in this cell." - " See https://nvcc4jupyter.readthedocs.io/en/latest/magics.html#cuda" + " See https://nvcc4jupyter.readthedocs.io/en/latest/magics.html#cuda" # noqa: E501 " for usage details." ) ) @@ -20,7 +20,7 @@ def get_parser_cuda_group_run() -> argparse.ArgumentParser: parser.description = ( "%%cuda_group_run magic that compiles and runs source files in a given" " group. See" - " https://nvcc4jupyter.readthedocs.io/en/latest/magics.html#cuda-group-run" + " https://nvcc4jupyter.readthedocs.io/en/latest/magics.html#cuda-group-run" # noqa: E501 " for usage details." ) parser.add_argument("-g", "--group", type=str, required=True) @@ -32,7 +32,7 @@ def get_parser_cuda_group_save() -> argparse.ArgumentParser: description=( "%%cuda_group_save magic that saves CUDA C++ code in this cell for" " later compilation and execution with possibly more source files." - " See https://nvcc4jupyter.readthedocs.io/en/latest/magics.html#cuda-group-save" + " See https://nvcc4jupyter.readthedocs.io/en/latest/magics.html#cuda-group-save" # noqa: E501 " for usage details." ) ) @@ -45,7 +45,7 @@ def get_parser_cuda_group_delete() -> argparse.ArgumentParser: parser = argparse.ArgumentParser( description=( "%%cuda_group_delete magic that deletes all files in a group. See" - " https://nvcc4jupyter.readthedocs.io/en/latest/magics.html#cuda-group-delete" + " https://nvcc4jupyter.readthedocs.io/en/latest/magics.html#cuda-group-delete" # noqa: E501 " for usage details." ) ) diff --git a/nvcc4jupyter/plugin.py b/nvcc4jupyter/plugin.py index e278d56..a4ff704 100644 --- a/nvcc4jupyter/plugin.py +++ b/nvcc4jupyter/plugin.py @@ -17,8 +17,8 @@ SHARED_GROUP_NAME = "shared" def print_out(out: str): - for l in out.split("\n"): - print(l) + for line in out.split("\n"): + print(line) @magics_class diff --git a/pyproject.toml b/pyproject.toml index 2e5ca54..5f5e495 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -85,28 +85,6 @@ experimental-string-processing = true [tool.coverage.run] branch = true -[tool.flake8] -max-line-length = 79 -select = "F,E,W,B,B901,B902,B903" -exclude = [ - ".eggs", - ".git", - ".tox", - "nssm", - "obj", - "out", - "packages", - "pywin32", - "tests", - "swagger_client" -] -ignore = [ - "E722", - "B001", - "W503", - "E203" -] - [tool.pyright] include = ["src"] exclude = [ diff --git a/tests/conftest.py b/tests/conftest.py index 634fb8c..3bb2d59 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -1 +1 @@ -from .fixtures.fixtures import * +from .fixtures.fixtures import * # noqa: F401,F403 diff --git a/tests/fixtures/fixtures.py b/tests/fixtures/fixtures.py index f97b086..93b88fb 100644 --- a/tests/fixtures/fixtures.py +++ b/tests/fixtures/fixtures.py @@ -30,7 +30,7 @@ def fixtures_path(tests_path): @pytest.fixture(scope="session") def sample_magic_cu_line(): # fmt: off - return '--profile --profiler-args "--metrics l1tex__t_sectors_pipe_lsu_mem_global_op_ld.sum"' + return '--profile --profiler-args "--metrics l1tex__t_sectors_pipe_lsu_mem_global_op_ld.sum"' # noqa: E501 # fmt: on From be6b7a01d647ea4ceb7de6d874b95cc794df5961 Mon Sep 17 00:00:00 2001 From: Cosmin Ciocan Date: Fri, 12 Jan 2024 18:12:49 +0100 Subject: [PATCH 40/46] Add pylint pre-commit hook --- .pre-commit-config.yaml | 7 +++++++ nvcc4jupyter/__init__.py | 4 ++++ nvcc4jupyter/parsers.py | 16 ++++++++++++++++ nvcc4jupyter/plugin.py | 17 +++++++++++++++-- pyproject.toml | 1 + 5 files changed, 43 insertions(+), 2 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 46ee56e..4cfc67f 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -37,3 +37,10 @@ repos: hooks: - id: flake8 args: ["--config", ".flake8"] + + # pylint check + - repo: https://github.com/pycqa/pylint + rev: v3.0.3 + hooks: + - id: pylint + args: ["--rcfile", "pyproject.toml"] diff --git a/nvcc4jupyter/__init__.py b/nvcc4jupyter/__init__.py index e9f7145..70a978f 100644 --- a/nvcc4jupyter/__init__.py +++ b/nvcc4jupyter/__init__.py @@ -1,3 +1,7 @@ +""" +nvcc4jupyter: CUDA C++ plugin for Jupyter Notebook +""" + from .plugin import NVCCPlugin, load_ipython_extension # noqa: F401 __version__ = "1.0.2" diff --git a/nvcc4jupyter/parsers.py b/nvcc4jupyter/parsers.py index 7679465..e94afce 100644 --- a/nvcc4jupyter/parsers.py +++ b/nvcc4jupyter/parsers.py @@ -1,7 +1,14 @@ +""" +Parsers for the CUDA magic commands. +""" + import argparse def get_parser_cuda() -> argparse.ArgumentParser: + """ + %%cuda magic command parser. + """ parser = argparse.ArgumentParser( description=( "%%cuda magic that compiles and runs CUDA C++ code in this cell." @@ -16,6 +23,9 @@ def get_parser_cuda() -> argparse.ArgumentParser: def get_parser_cuda_group_run() -> argparse.ArgumentParser: + """ + %%cuda_group_run magic command parser. + """ parser = get_parser_cuda() parser.description = ( "%%cuda_group_run magic that compiles and runs source files in a given" @@ -28,6 +38,9 @@ def get_parser_cuda_group_run() -> argparse.ArgumentParser: def get_parser_cuda_group_save() -> argparse.ArgumentParser: + """ + %%cuda_group_save magic command parser. + """ parser = argparse.ArgumentParser( description=( "%%cuda_group_save magic that saves CUDA C++ code in this cell for" @@ -42,6 +55,9 @@ def get_parser_cuda_group_save() -> argparse.ArgumentParser: def get_parser_cuda_group_delete() -> argparse.ArgumentParser: + """ + %%cuda_group_delete magic command parser. + """ parser = argparse.ArgumentParser( description=( "%%cuda_group_delete magic that deletes all files in a group. See" diff --git a/nvcc4jupyter/plugin.py b/nvcc4jupyter/plugin.py index a4ff704..269a2cc 100644 --- a/nvcc4jupyter/plugin.py +++ b/nvcc4jupyter/plugin.py @@ -1,3 +1,7 @@ +""" +nvcc4jupyter: CUDA C++ plugin for Jupyter Notebook +""" + import argparse import glob import os @@ -7,6 +11,7 @@ import tempfile import uuid from typing import List, Optional +# pylint: disable=import-error from IPython.core.interactiveshell import InteractiveShell from IPython.core.magic import Magics, cell_magic, line_magic, magics_class @@ -17,14 +22,19 @@ SHARED_GROUP_NAME = "shared" def print_out(out: str): + """Print string line by line.""" for line in out.split("\n"): print(line) @magics_class class NVCCPlugin(Magics): + """ + CUDA C++ plugin for Jupyter Notebook + """ + def __init__(self, shell: InteractiveShell): - super(NVCCPlugin, self).__init__(shell) + super().__init__(shell) self.shell: InteractiveShell # type hint not provided by parent class self.parser_cuda = parsers.get_parser_cuda() @@ -55,7 +65,7 @@ class NVCCPlugin(Magics): ValueError: If the source name does not have a proper extension. """ _, ext = os.path.splitext(source_name) - if ext != ".cu" and ext != ".h": + if ext not in (".cu", ".h"): raise ValueError( f'Given source name "{source_name}" must end in ".h" or ".cu".' ) @@ -304,5 +314,8 @@ class NVCCPlugin(Magics): def load_ipython_extension(shell: InteractiveShell): + """ + Method used by IPython to load the extension. + """ nvcc_plugin = NVCCPlugin(shell) shell.register_magics(nvcc_plugin) diff --git a/pyproject.toml b/pyproject.toml index 5f5e495..ecad9fb 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -142,6 +142,7 @@ extension-pkg-whitelist= [ ] ignore="CVS" ignore-patterns="test.*?py,conftest.py" +ignore-paths="docs,tests" init-hook='import sys; sys.setrecursionlimit(8 * sys.getrecursionlimit())' jobs=0 limit-inference-results=100 From 063fe0015c65599cf6be0c4a2320d8fa1a830568 Mon Sep 17 00:00:00 2001 From: Cosmin Ciocan Date: Fri, 12 Jan 2024 18:33:43 +0100 Subject: [PATCH 41/46] Add bandit pre-commit hook to check for security issues --- .pre-commit-config.yaml | 11 +++++++++-- pyproject.toml | 4 +++- 2 files changed, 12 insertions(+), 3 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 4cfc67f..de41dce 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -19,7 +19,7 @@ repos: # python code formatting - repo: https://github.com/psf/black - rev: 23.1.0 + rev: 23.12.1 hooks: - id: black args: ["--config", "pyproject.toml"] @@ -33,7 +33,7 @@ repos: # python check (PEP8), programming errors and code complexity - repo: https://github.com/PyCQA/flake8 - rev: 6.0.0 + rev: 7.0.0 hooks: - id: flake8 args: ["--config", ".flake8"] @@ -44,3 +44,10 @@ repos: hooks: - id: pylint args: ["--rcfile", "pyproject.toml"] + + - repo: https://github.com/PyCQA/bandit + rev: 1.7.6 + hooks: + - id: bandit + args: ["-c", "pyproject.toml"] + additional_dependencies: ["bandit[toml]"] diff --git a/pyproject.toml b/pyproject.toml index ecad9fb..71966ef 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -75,7 +75,9 @@ exclude_dirs = ["build","dist","tests","scripts"] number = 4 recursive = true targets = "src" -skips = ["B101", "B311"] +# B404 and B603 are skipped because the user can already run any arbitrary +# command on their jupyter server +skips = ["B101", "B311", "B404", "B603"] [tool.black] line-length = 79 From df7fc2ebebe4ac7c25f7a96cf2a6dcbdb4f6175b Mon Sep 17 00:00:00 2001 From: Cosmin Ciocan Date: Fri, 12 Jan 2024 18:44:03 +0100 Subject: [PATCH 42/46] Add github workflow to check pre-commit hooks on all files on master branch --- .github/workflows/code-quality-master.yml | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) create mode 100644 .github/workflows/code-quality-master.yml diff --git a/.github/workflows/code-quality-master.yml b/.github/workflows/code-quality-master.yml new file mode 100644 index 0000000..ab4f9f0 --- /dev/null +++ b/.github/workflows/code-quality-master.yml @@ -0,0 +1,22 @@ +# Same as `code-quality-pr.yaml` but triggered on commit to master branch +# and runs on all files (instead of only the changed ones) + +name: Code Quality Master + +on: + push: + branches: [master] + +jobs: + code-quality: + runs-on: ubuntu-latest + + steps: + - name: Checkout + uses: actions/checkout@v2 + + - name: Set up Python + uses: actions/setup-python@v2 + + - name: Run pre-commits + uses: pre-commit/action@v2.0.3 From 4521763395c459306fd1e880541a19632ceb6f94 Mon Sep 17 00:00:00 2001 From: Cosmin Ciocan Date: Fri, 12 Jan 2024 18:47:49 +0100 Subject: [PATCH 43/46] Add github workflow to check pre-commit hooks on modified files on pull requests --- .github/workflows/code-quality-pr.yml | 36 +++++++++++++++++++++++++++ 1 file changed, 36 insertions(+) create mode 100644 .github/workflows/code-quality-pr.yml diff --git a/.github/workflows/code-quality-pr.yml b/.github/workflows/code-quality-pr.yml new file mode 100644 index 0000000..4bffba5 --- /dev/null +++ b/.github/workflows/code-quality-pr.yml @@ -0,0 +1,36 @@ +# This workflow finds which files were changed, prints them, +# and runs `pre-commit` on those files. + +# Inspired by the sktime library: +# https://github.com/alan-turing-institute/sktime/blob/main/.github/workflows/test.yml + +name: Code Quality PR + +on: + pull_request: + branches: [master, "release/*", "dev"] + +jobs: + code-quality: + runs-on: ubuntu-latest + + steps: + - name: Checkout + uses: actions/checkout@v2 + + - name: Set up Python + uses: actions/setup-python@v2 + + - name: Find modified files + id: file_changes + uses: trilom/file-changes-action@v1.2.4 + with: + output: " " + + - name: List modified files + run: echo '${{ steps.file_changes.outputs.files}}' + + - name: Run pre-commits + uses: pre-commit/action@v2.0.3 + with: + extra_args: --files ${{ steps.file_changes.outputs.files}} From ed0e3a721cf0077069335c3d23db8b163bcf9c71 Mon Sep 17 00:00:00 2001 From: Cosmin Ciocan Date: Fri, 12 Jan 2024 18:59:30 +0100 Subject: [PATCH 44/46] Add code quality badges to README.md --- README.md | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/README.md b/README.md index f82f00f..f60b626 100644 --- a/README.md +++ b/README.md @@ -3,13 +3,23 @@ | | | | --- | --- | | Testing | ![Python Versions][python-version] [![CI - Test][test-badge]][test-workflow] [![Coverage][coverage-badge]][coverage-results] | +| Code Quality | [![Code style: black][black-badge]][black-project] [![security: bandit][bandit-badge]][bandit-project]| | Package | [![PyPI Latest Release][pypi-latest-version]][pypi-project-url] [![PyPI Downloads][pypi-downloads]][pypi-project-url] | + [python-version]: https://img.shields.io/pypi/pyversions/nvcc4jupyter [test-badge]: https://github.com/cosminc98/nvcc4jupyter/actions/workflows/test.yml/badge.svg [test-workflow]: https://github.com/cosminc98/nvcc4jupyter/actions/workflows/test.yml [coverage-badge]: https://codecov.io/github/cosminc98/nvcc4jupyter/coverage.svg?branch=master [coverage-results]: https://codecov.io/gh/cosminc98/nvcc4jupyter + + +[black-badge]: https://img.shields.io/badge/code%20style-black-000000.svg +[black-project]: https://github.com/ambv/black +[bandit-badge]: https://img.shields.io/badge/security-bandit-yellow.svg +[bandit-project]: https://github.com/PyCQA/bandit + + [pypi-project-url]: https://pypi.org/project/nvcc4jupyter/ [pypi-latest-version]: https://img.shields.io/pypi/v/nvcc4jupyter.svg [pypi-downloads]: https://img.shields.io/pypi/dm/nvcc4jupyter.svg?label=PyPI%20downloads From 0601610905fcc876164f5d45a3711f4491de5694 Mon Sep 17 00:00:00 2001 From: Cosmin Ciocan Date: Fri, 12 Jan 2024 19:00:22 +0100 Subject: [PATCH 45/46] Change to version 1.0.3 --- nvcc4jupyter/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nvcc4jupyter/__init__.py b/nvcc4jupyter/__init__.py index 70a978f..97b8902 100644 --- a/nvcc4jupyter/__init__.py +++ b/nvcc4jupyter/__init__.py @@ -4,4 +4,4 @@ nvcc4jupyter: CUDA C++ plugin for Jupyter Notebook from .plugin import NVCCPlugin, load_ipython_extension # noqa: F401 -__version__ = "1.0.2" +__version__ = "1.0.3" From e392fc382b3017ef5409c138959416286337fa45 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Cosmin=20=C8=98tefan=20Ciocan?= <57830279+cosminc98@users.noreply.github.com> Date: Sat, 13 Jan 2024 00:46:30 +0000 Subject: [PATCH 46/46] Create devcontainer for quick dev setups --- .devcontainer/Dockerfile | 19 +++++++++++++++++++ .devcontainer/devcontainer.json | 26 ++++++++++++++++++++++++++ .devcontainer/post_create.sh | 7 +++++++ 3 files changed, 52 insertions(+) create mode 100644 .devcontainer/Dockerfile create mode 100644 .devcontainer/devcontainer.json create mode 100644 .devcontainer/post_create.sh diff --git a/.devcontainer/Dockerfile b/.devcontainer/Dockerfile new file mode 100644 index 0000000..9088efc --- /dev/null +++ b/.devcontainer/Dockerfile @@ -0,0 +1,19 @@ +FROM ubuntu + +ARG VENV_PATH=/opt/dev-venv +ENV VENV_ACTIVATE=${VENV_PATH}/bin/activate + +RUN apt update +RUN apt install -y python3.10-venv nvidia-cuda-toolkit gcc vim git + +# the mkdir command bypasses a profiler error, which allows us to run it with +# host code only to at least check that the profiler parameters are correctly +# provided; without this line, some tests will fail +RUN mkdir -p /usr/lib/x86_64-linux-gnu/nsight-compute/sections + +# we create the virtualenv here so that the devcontainer.json setting +# python.defaultInterpreterPath can be used to find it; if we do it in the +# post_create.sh script, the virtualenv will not be loaded and features like +# pylance, black, isort, etc. will not work +RUN python3.10 -m venv ${VENV_PATH} +RUN echo "source ${VENV_ACTIVATE}" >> ~/.bashrc diff --git a/.devcontainer/devcontainer.json b/.devcontainer/devcontainer.json new file mode 100644 index 0000000..c6e997c --- /dev/null +++ b/.devcontainer/devcontainer.json @@ -0,0 +1,26 @@ +{ + "name": "Python Environment", + "build": { + "dockerfile": "Dockerfile", + "context": ".." + }, + "postCreateCommand": "bash .devcontainer/post_create.sh", + "customizations": { + "vscode": { + "extensions": [ + "editorconfig.editorconfig", + "ms-azuretools.vscode-docker", + "ms-python.python", + "ms-python.vscode-pylance", + "ms-python.pylint", + "ms-python.isort", + "ms-python.flake8", + "ms-python.black-formatter", + "ryanluker.vscode-coverage-gutters" + ], + "settings": { + "python.defaultInterpreterPath": "/opt/dev-venv/bin/python" + } + } + } +} diff --git a/.devcontainer/post_create.sh b/.devcontainer/post_create.sh new file mode 100644 index 0000000..15fd069 --- /dev/null +++ b/.devcontainer/post_create.sh @@ -0,0 +1,7 @@ +#!/bin/bash + +# install developer dependencies +pip install .[dev] + +# make sure the developer uses pre-commit hooks +pre-commit install