From 1464d5ca43273ccf7a7add66a211f67a982ae9f3 Mon Sep 17 00:00:00 2001 From: Terry Tianlin Luan <44815845+terryluan12@users.noreply.github.com> Date: Mon, 29 Dec 2025 23:10:14 -0500 Subject: [PATCH 0001/1575] Adding + Fixing Clippy rules to better align with #[no_std] (#6570) * * Added alloc_instead_of_core, std_instead_of_alloc, and std_instead_of_core clippy rules * Manually changed part of the code to use core/alloc * use clippy --fix to fix issues in stdlib * * Used clippy --fix to fix issues in vm * Imported Range in vm/src/anystr.rs * * Used clippy --fix to fix issues in common --- Cargo.toml | 3 + crates/codegen/src/compile.rs | 13 +-- crates/codegen/src/error.rs | 5 +- crates/codegen/src/ir.rs | 6 +- crates/codegen/src/lib.rs | 2 + crates/codegen/src/string_parser.rs | 4 +- crates/codegen/src/symboltable.rs | 12 +-- crates/codegen/src/unparse.rs | 9 +- crates/common/src/borrow.rs | 6 +- crates/common/src/boxvec.rs | 12 +-- crates/common/src/cformat.rs | 15 ++-- crates/common/src/crt_fd.rs | 4 +- crates/common/src/encodings.rs | 4 +- crates/common/src/fileutils.rs | 10 +-- crates/common/src/format.rs | 6 +- crates/common/src/hash.rs | 8 +- crates/common/src/int.rs | 12 +-- crates/common/src/lib.rs | 2 + crates/common/src/linked_list.rs | 2 +- crates/common/src/lock/cell_lock.rs | 2 +- crates/common/src/lock/immutable_mutex.rs | 7 +- crates/common/src/lock/thread_mutex.rs | 14 +-- crates/common/src/os.rs | 3 +- crates/common/src/rc.rs | 4 +- crates/common/src/str.rs | 25 +++--- crates/compiler-core/src/bytecode.rs | 13 +-- crates/compiler-core/src/lib.rs | 2 + crates/compiler-core/src/marshal.rs | 18 ++-- crates/compiler-core/src/mode.rs | 6 +- crates/compiler/src/lib.rs | 4 +- crates/derive-impl/src/compile_bytecode.rs | 4 +- crates/derive-impl/src/from_args.rs | 8 +- crates/derive-impl/src/pyclass.rs | 10 +-- crates/derive-impl/src/pymodule.rs | 9 +- crates/derive-impl/src/pytraverse.rs | 2 +- crates/derive-impl/src/util.rs | 10 +-- crates/derive/src/lib.rs | 2 +- crates/jit/src/lib.rs | 5 +- crates/literal/src/escape.rs | 30 +++---- crates/literal/src/float.rs | 2 +- crates/sre_engine/src/engine.rs | 8 +- crates/sre_engine/src/string.rs | 2 +- crates/stdlib/src/array.rs | 41 ++++----- crates/stdlib/src/binascii.rs | 2 +- crates/stdlib/src/bz2.rs | 3 +- crates/stdlib/src/cmath.rs | 4 +- crates/stdlib/src/compression.rs | 4 +- crates/stdlib/src/contextvars.rs | 26 +++--- crates/stdlib/src/csv.rs | 7 +- crates/stdlib/src/faulthandler.rs | 25 +++--- crates/stdlib/src/fcntl.rs | 2 +- crates/stdlib/src/grp.rs | 4 +- crates/stdlib/src/hashlib.rs | 8 +- crates/stdlib/src/json.rs | 4 +- crates/stdlib/src/lib.rs | 3 +- crates/stdlib/src/locale.rs | 6 +- crates/stdlib/src/lzma.rs | 2 +- crates/stdlib/src/math.rs | 16 ++-- crates/stdlib/src/mmap.rs | 4 +- crates/stdlib/src/opcode.rs | 2 +- crates/stdlib/src/openssl.rs | 8 +- crates/stdlib/src/overlapped.rs | 24 ++--- crates/stdlib/src/posixshmem.rs | 2 +- crates/stdlib/src/posixsubprocess.rs | 10 +-- crates/stdlib/src/resource.rs | 3 +- crates/stdlib/src/scproxy.rs | 2 +- crates/stdlib/src/select.rs | 15 ++-- crates/stdlib/src/socket.rs | 32 ++++--- crates/stdlib/src/sqlite.rs | 4 +- crates/stdlib/src/ssl.rs | 22 +++-- crates/stdlib/src/ssl/cert.rs | 31 +++---- crates/stdlib/src/ssl/compat.rs | 3 +- crates/stdlib/src/syslog.rs | 5 +- crates/stdlib/src/tkinter.rs | 8 +- crates/stdlib/src/zlib.rs | 4 +- crates/venvlauncher/src/main.rs | 6 +- crates/vm/src/anystr.rs | 30 +++---- crates/vm/src/buffer.rs | 12 +-- crates/vm/src/builtins/bool.rs | 4 +- crates/vm/src/builtins/builtin_func.rs | 4 +- crates/vm/src/builtins/bytearray.rs | 4 +- crates/vm/src/builtins/bytes.rs | 2 +- crates/vm/src/builtins/code.rs | 5 +- crates/vm/src/builtins/complex.rs | 2 +- crates/vm/src/builtins/descriptor.rs | 12 +-- crates/vm/src/builtins/dict.rs | 6 +- crates/vm/src/builtins/function.rs | 4 +- crates/vm/src/builtins/genericalias.rs | 2 +- crates/vm/src/builtins/getset.rs | 6 +- crates/vm/src/builtins/int.rs | 8 +- crates/vm/src/builtins/list.rs | 17 ++-- crates/vm/src/builtins/map.rs | 2 +- crates/vm/src/builtins/memory.rs | 2 +- crates/vm/src/builtins/module.rs | 4 +- crates/vm/src/builtins/object.rs | 2 +- crates/vm/src/builtins/property.rs | 4 +- crates/vm/src/builtins/range.rs | 2 +- crates/vm/src/builtins/set.rs | 31 +++---- crates/vm/src/builtins/str.rs | 43 ++++----- crates/vm/src/builtins/traceback.rs | 2 +- crates/vm/src/builtins/tuple.rs | 27 +++--- crates/vm/src/builtins/type.rs | 27 +++--- crates/vm/src/builtins/union.rs | 2 +- crates/vm/src/bytes_inner.rs | 14 +-- crates/vm/src/cformat.rs | 4 +- crates/vm/src/class.rs | 4 +- crates/vm/src/codecs.rs | 8 +- crates/vm/src/convert/try_from.rs | 2 +- crates/vm/src/dict_inner.rs | 9 +- crates/vm/src/exceptions.rs | 10 +-- crates/vm/src/frame.rs | 25 +++--- crates/vm/src/function/argument.rs | 8 +- crates/vm/src/function/builtin.rs | 8 +- crates/vm/src/function/either.rs | 2 +- crates/vm/src/function/fspath.rs | 7 +- crates/vm/src/function/method.rs | 6 +- crates/vm/src/function/number.rs | 4 +- crates/vm/src/function/protocol.rs | 12 +-- crates/vm/src/intern.rs | 30 +++---- crates/vm/src/lib.rs | 1 + crates/vm/src/macros.rs | 4 +- crates/vm/src/object/core.rs | 52 +++++------ crates/vm/src/object/ext.rs | 9 +- crates/vm/src/object/payload.rs | 18 ++-- crates/vm/src/object/traverse.rs | 2 +- crates/vm/src/object/traverse_object.rs | 2 +- crates/vm/src/ospath.rs | 6 +- crates/vm/src/protocol/buffer.rs | 11 +-- crates/vm/src/protocol/callable.rs | 4 +- crates/vm/src/protocol/iter.rs | 8 +- crates/vm/src/protocol/mapping.rs | 8 +- crates/vm/src/protocol/number.rs | 2 +- crates/vm/src/protocol/sequence.rs | 8 +- crates/vm/src/py_io.rs | 4 +- crates/vm/src/py_serde.rs | 2 +- crates/vm/src/readline.rs | 2 +- crates/vm/src/scope.rs | 2 +- crates/vm/src/sequence.rs | 4 +- crates/vm/src/signal.rs | 10 +-- crates/vm/src/sliceable.rs | 2 +- crates/vm/src/stdlib/ast/elif_else_clause.rs | 2 +- crates/vm/src/stdlib/ast/parameter.rs | 2 +- crates/vm/src/stdlib/ast/string.rs | 4 +- crates/vm/src/stdlib/atexit.rs | 2 +- crates/vm/src/stdlib/builtins.rs | 10 +-- crates/vm/src/stdlib/codecs.rs | 8 +- crates/vm/src/stdlib/collections.rs | 10 +-- crates/vm/src/stdlib/ctypes.rs | 42 +++++---- crates/vm/src/stdlib/ctypes/array.rs | 14 +-- crates/vm/src/stdlib/ctypes/base.rs | 64 ++++++------- crates/vm/src/stdlib/ctypes/function.rs | 50 +++++------ crates/vm/src/stdlib/ctypes/library.rs | 4 +- crates/vm/src/stdlib/ctypes/pointer.rs | 56 ++++++------ crates/vm/src/stdlib/ctypes/simple.rs | 94 ++++++++++---------- crates/vm/src/stdlib/ctypes/structure.rs | 12 +-- crates/vm/src/stdlib/ctypes/union.rs | 12 +-- crates/vm/src/stdlib/io.rs | 52 +++++------ crates/vm/src/stdlib/itertools.rs | 4 +- crates/vm/src/stdlib/mod.rs | 3 +- crates/vm/src/stdlib/nt.rs | 34 +++---- crates/vm/src/stdlib/os.rs | 29 +++--- crates/vm/src/stdlib/posix.rs | 14 +-- crates/vm/src/stdlib/pwd.rs | 4 +- crates/vm/src/stdlib/signal.rs | 14 +-- crates/vm/src/stdlib/string.rs | 2 +- crates/vm/src/stdlib/symtable.rs | 4 +- crates/vm/src/stdlib/sys.rs | 12 +-- crates/vm/src/stdlib/thread.rs | 8 +- crates/vm/src/stdlib/time.rs | 28 +++--- crates/vm/src/stdlib/winapi.rs | 12 +-- crates/vm/src/stdlib/winreg.rs | 18 ++-- crates/vm/src/suggestion.rs | 2 +- crates/vm/src/types/slot.rs | 18 ++-- crates/vm/src/utils.rs | 8 +- crates/vm/src/version.rs | 3 +- crates/vm/src/vm/context.rs | 6 +- crates/vm/src/vm/interpreter.rs | 2 +- crates/vm/src/vm/mod.rs | 16 ++-- crates/vm/src/vm/thread.rs | 6 +- crates/vm/src/vm/vm_new.rs | 4 +- crates/vm/src/vm/vm_ops.rs | 4 +- crates/vm/src/warn.rs | 2 +- crates/vm/src/windows.rs | 6 +- examples/dis.rs | 2 +- src/lib.rs | 2 +- 185 files changed, 1000 insertions(+), 952 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index 06734b47e18..12607cfa2cc 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -232,6 +232,9 @@ unsafe_op_in_unsafe_fn = "deny" elided_lifetimes_in_paths = "warn" [workspace.lints.clippy] +# alloc_instead_of_core = "warn" +# std_instead_of_alloc = "warn" +# std_instead_of_core = "warn" perf = "warn" style = "warn" complexity = "warn" diff --git a/crates/codegen/src/compile.rs b/crates/codegen/src/compile.rs index 7909e924251..c44b2b00684 100644 --- a/crates/codegen/src/compile.rs +++ b/crates/codegen/src/compile.rs @@ -16,6 +16,7 @@ use crate::{ symboltable::{self, CompilerScope, SymbolFlags, SymbolScope, SymbolTable}, unparse::UnparseExpr, }; +use alloc::borrow::Cow; use itertools::Itertools; use malachite_bigint::BigInt; use num_complex::Complex; @@ -42,7 +43,7 @@ use rustpython_compiler_core::{ }, }; use rustpython_wtf8::Wtf8Buf; -use std::{borrow::Cow, collections::HashSet}; +use std::collections::HashSet; const MAXBLOCKS: usize = 20; @@ -293,7 +294,7 @@ fn compiler_unwrap_option(zelf: &Compiler, o: Option) -> T { o.unwrap() } -// fn compiler_result_unwrap(zelf: &Compiler, result: Result) -> T { +// fn compiler_result_unwrap(zelf: &Compiler, result: Result) -> T { // if result.is_err() { // eprintln!("=== CODEGEN PANIC INFO ==="); // eprintln!("This IS an internal error, an result was unwrapped during codegen"); @@ -1831,7 +1832,7 @@ impl Compiler { name.to_owned(), ); - let args_iter = std::iter::empty() + let args_iter = core::iter::empty() .chain(¶meters.posonlyargs) .chain(¶meters.args) .map(|arg| &arg.parameter) @@ -2438,7 +2439,7 @@ impl Compiler { let mut funcflags = bytecode::MakeFunctionFlags::empty(); // Handle positional defaults - let defaults: Vec<_> = std::iter::empty() + let defaults: Vec<_> = core::iter::empty() .chain(¶meters.posonlyargs) .chain(¶meters.args) .filter_map(|x| x.default.as_deref()) @@ -2566,7 +2567,7 @@ impl Compiler { let mut num_annotations = 0; // Handle parameter annotations - let parameters_iter = std::iter::empty() + let parameters_iter = core::iter::empty() .chain(¶meters.posonlyargs) .chain(¶meters.args) .chain(¶meters.kwonlyargs) @@ -4965,7 +4966,7 @@ impl Compiler { let name = "".to_owned(); // Prepare defaults before entering function - let defaults: Vec<_> = std::iter::empty() + let defaults: Vec<_> = core::iter::empty() .chain(¶ms.posonlyargs) .chain(¶ms.args) .filter_map(|x| x.default.as_deref()) diff --git a/crates/codegen/src/error.rs b/crates/codegen/src/error.rs index 70e2f13f253..459ba8e33b5 100644 --- a/crates/codegen/src/error.rs +++ b/crates/codegen/src/error.rs @@ -1,5 +1,6 @@ +use alloc::fmt; +use core::fmt::Display; use rustpython_compiler_core::SourceLocation; -use std::fmt::{self, Display}; use thiserror::Error; #[derive(Debug)] @@ -93,7 +94,7 @@ pub enum CodegenErrorType { NotImplementedYet, // RustPython marker for unimplemented features } -impl std::error::Error for CodegenErrorType {} +impl core::error::Error for CodegenErrorType {} impl fmt::Display for CodegenErrorType { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { diff --git a/crates/codegen/src/ir.rs b/crates/codegen/src/ir.rs index de0126f1122..670635fbd37 100644 --- a/crates/codegen/src/ir.rs +++ b/crates/codegen/src/ir.rs @@ -1,4 +1,4 @@ -use std::ops; +use core::ops; use crate::{IndexMap, IndexSet, error::InternalError}; use rustpython_compiler_core::{ @@ -198,7 +198,7 @@ impl CodeInfo { *arg = new_arg; } let (extras, lo_arg) = arg.split(); - locations.extend(std::iter::repeat_n(info.location, arg.instr_size())); + locations.extend(core::iter::repeat_n(info.location, arg.instr_size())); instructions.extend( extras .map(|byte| CodeUnit::new(Instruction::ExtendedArg, byte)) @@ -401,7 +401,7 @@ fn stackdepth_push( fn iter_blocks(blocks: &[Block]) -> impl Iterator + '_ { let mut next = BlockIdx(0); - std::iter::from_fn(move || { + core::iter::from_fn(move || { if next == BlockIdx::NULL { return None; } diff --git a/crates/codegen/src/lib.rs b/crates/codegen/src/lib.rs index 291b57d7f67..34d3870ae91 100644 --- a/crates/codegen/src/lib.rs +++ b/crates/codegen/src/lib.rs @@ -5,6 +5,8 @@ #[macro_use] extern crate log; +extern crate alloc; + type IndexMap = indexmap::IndexMap; type IndexSet = indexmap::IndexSet; diff --git a/crates/codegen/src/string_parser.rs b/crates/codegen/src/string_parser.rs index ede2f118c37..175e75c1a26 100644 --- a/crates/codegen/src/string_parser.rs +++ b/crates/codegen/src/string_parser.rs @@ -5,7 +5,7 @@ //! after ruff has already successfully parsed the string literal, meaning //! we don't need to do any validation or error handling. -use std::convert::Infallible; +use core::convert::Infallible; use ruff_python_ast::{AnyStringFlags, StringFlags}; use rustpython_wtf8::{CodePoint, Wtf8, Wtf8Buf}; @@ -96,7 +96,7 @@ impl StringParser { } // OK because radix_bytes is always going to be in the ASCII range. - let radix_str = std::str::from_utf8(&radix_bytes[..len]).expect("ASCII bytes"); + let radix_str = core::str::from_utf8(&radix_bytes[..len]).expect("ASCII bytes"); let value = u32::from_str_radix(radix_str, 8).unwrap(); char::from_u32(value).unwrap() } diff --git a/crates/codegen/src/symboltable.rs b/crates/codegen/src/symboltable.rs index 3c8454b9e22..1629e5fff38 100644 --- a/crates/codegen/src/symboltable.rs +++ b/crates/codegen/src/symboltable.rs @@ -11,6 +11,7 @@ use crate::{ IndexMap, error::{CodegenError, CodegenErrorType}, }; +use alloc::{borrow::Cow, fmt}; use bitflags::bitflags; use ruff_python_ast::{ self as ast, Comprehension, Decorator, Expr, Identifier, ModExpression, ModModule, Parameter, @@ -20,7 +21,6 @@ use ruff_python_ast::{ }; use ruff_text_size::{Ranged, TextRange}; use rustpython_compiler_core::{PositionEncoding, SourceFile, SourceLocation}; -use std::{borrow::Cow, fmt}; /// Captures all symbols in the current scope, and has a list of sub-scopes in this scope. #[derive(Clone)] @@ -215,8 +215,8 @@ impl SymbolTableError { type SymbolTableResult = Result; -impl std::fmt::Debug for SymbolTable { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { +impl core::fmt::Debug for SymbolTable { + fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result { write!( f, "SymbolTable({:?} symbols, {:?} sub scopes)", @@ -261,8 +261,8 @@ fn drop_class_free(symbol_table: &mut SymbolTable) { type SymbolMap = IndexMap; mod stack { + use core::ptr::NonNull; use std::panic; - use std::ptr::NonNull; pub struct StackStack { v: Vec>, } @@ -325,7 +325,7 @@ struct SymbolTableAnalyzer { impl SymbolTableAnalyzer { fn analyze_symbol_table(&mut self, symbol_table: &mut SymbolTable) -> SymbolTableResult { - let symbols = std::mem::take(&mut symbol_table.symbols); + let symbols = core::mem::take(&mut symbol_table.symbols); let sub_tables = &mut *symbol_table.sub_tables; let mut info = (symbols, symbol_table.typ); @@ -689,7 +689,7 @@ impl SymbolTableBuilder { fn leave_scope(&mut self) { let mut table = self.tables.pop().unwrap(); // Save the collected varnames to the symbol table - table.varnames = std::mem::take(&mut self.current_varnames); + table.varnames = core::mem::take(&mut self.current_varnames); self.tables.last_mut().unwrap().sub_tables.push(table); } diff --git a/crates/codegen/src/unparse.rs b/crates/codegen/src/unparse.rs index 74e35fd5e2a..7b26d229187 100644 --- a/crates/codegen/src/unparse.rs +++ b/crates/codegen/src/unparse.rs @@ -1,3 +1,5 @@ +use alloc::fmt; +use core::fmt::Display as _; use ruff_python_ast::{ self as ruff, Arguments, BoolOp, Comprehension, ConversionFlag, Expr, Identifier, Operator, Parameter, ParameterWithDefault, Parameters, @@ -5,7 +7,6 @@ use ruff_python_ast::{ use ruff_text_size::Ranged; use rustpython_compiler_core::SourceFile; use rustpython_literal::escape::{AsciiEscape, UnicodeEscape}; -use std::fmt::{self, Display as _}; mod precedence { macro_rules! precedence { @@ -51,7 +52,7 @@ impl<'a, 'b, 'c> Unparser<'a, 'b, 'c> { } fn p_delim(&mut self, first: &mut bool, s: &str) -> fmt::Result { - self.p_if(!std::mem::take(first), s) + self.p_if(!core::mem::take(first), s) } fn write_fmt(&mut self, f: fmt::Arguments<'_>) -> fmt::Result { @@ -575,7 +576,7 @@ impl<'a, 'b, 'c> Unparser<'a, 'b, 'c> { if conversion != ConversionFlag::None { self.p("!")?; let buf = &[conversion as u8]; - let c = std::str::from_utf8(buf).unwrap(); + let c = core::str::from_utf8(buf).unwrap(); self.p(c)?; } @@ -650,7 +651,7 @@ impl fmt::Display for UnparseExpr<'_> { } fn to_string_fmt(f: impl FnOnce(&mut fmt::Formatter<'_>) -> fmt::Result) -> String { - use std::cell::Cell; + use core::cell::Cell; struct Fmt(Cell>); impl) -> fmt::Result> fmt::Display for Fmt { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { diff --git a/crates/common/src/borrow.rs b/crates/common/src/borrow.rs index 610084006e1..d8389479b33 100644 --- a/crates/common/src/borrow.rs +++ b/crates/common/src/borrow.rs @@ -2,10 +2,8 @@ use crate::lock::{ MapImmutable, PyImmutableMappedMutexGuard, PyMappedMutexGuard, PyMappedRwLockReadGuard, PyMappedRwLockWriteGuard, PyMutexGuard, PyRwLockReadGuard, PyRwLockWriteGuard, }; -use std::{ - fmt, - ops::{Deref, DerefMut}, -}; +use alloc::fmt; +use core::ops::{Deref, DerefMut}; macro_rules! impl_from { ($lt:lifetime, $gen:ident, $t:ty, $($var:ident($from:ty),)*) => { diff --git a/crates/common/src/boxvec.rs b/crates/common/src/boxvec.rs index 8687ba7f7f5..3260e76ca87 100644 --- a/crates/common/src/boxvec.rs +++ b/crates/common/src/boxvec.rs @@ -2,13 +2,13 @@ //! An unresizable vector backed by a `Box<[T]>` #![allow(clippy::needless_lifetimes)] - -use std::{ +use alloc::{fmt, slice}; +use core::{ borrow::{Borrow, BorrowMut}, - cmp, fmt, + cmp, mem::{self, MaybeUninit}, ops::{Bound, Deref, DerefMut, RangeBounds}, - ptr, slice, + ptr, }; pub struct BoxVec { @@ -555,7 +555,7 @@ impl Extend for BoxVec { }; let mut iter = iter.into_iter(); loop { - if std::ptr::eq(ptr, end_ptr) { + if core::ptr::eq(ptr, end_ptr) { break; } if let Some(elt) = iter.next() { @@ -693,7 +693,7 @@ impl CapacityError { const CAPERROR: &str = "insufficient capacity"; -impl std::error::Error for CapacityError {} +impl core::error::Error for CapacityError {} impl fmt::Display for CapacityError { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { diff --git a/crates/common/src/cformat.rs b/crates/common/src/cformat.rs index b553f0b6b10..24332396fdb 100644 --- a/crates/common/src/cformat.rs +++ b/crates/common/src/cformat.rs @@ -1,15 +1,16 @@ //! Implementation of Printf-Style string formatting //! as per the [Python Docs](https://docs.python.org/3/library/stdtypes.html#printf-style-string-formatting). +use alloc::fmt; use bitflags::bitflags; +use core::{ + cmp, + iter::{Enumerate, Peekable}, + str::FromStr, +}; use itertools::Itertools; use malachite_bigint::{BigInt, Sign}; use num_traits::Signed; use rustpython_literal::{float, format::Case}; -use std::{ - cmp, fmt, - iter::{Enumerate, Peekable}, - str::FromStr, -}; use crate::wtf8::{CodePoint, Wtf8, Wtf8Buf}; @@ -785,7 +786,7 @@ impl CFormatStrOrBytes { if !literal.is_empty() { parts.push(( part_index, - CFormatPart::Literal(std::mem::take(&mut literal)), + CFormatPart::Literal(core::mem::take(&mut literal)), )); } let spec = CFormatSpecKeyed::parse(iter).map_err(|err| CFormatError { @@ -816,7 +817,7 @@ impl CFormatStrOrBytes { impl IntoIterator for CFormatStrOrBytes { type Item = (usize, CFormatPart); - type IntoIter = std::vec::IntoIter; + type IntoIter = alloc::vec::IntoIter; fn into_iter(self) -> Self::IntoIter { self.parts.into_iter() diff --git a/crates/common/src/crt_fd.rs b/crates/common/src/crt_fd.rs index b873ef9c52c..1902a362e32 100644 --- a/crates/common/src/crt_fd.rs +++ b/crates/common/src/crt_fd.rs @@ -1,7 +1,9 @@ //! A module implementing an io type backed by the C runtime's file descriptors, i.e. what's //! returned from libc::open, even on windows. -use std::{cmp, ffi, fmt, io}; +use alloc::fmt; +use core::cmp; +use std::{ffi, io}; #[cfg(not(windows))] use std::os::fd::{AsFd, AsRawFd, BorrowedFd, FromRawFd, IntoRawFd, OwnedFd, RawFd}; diff --git a/crates/common/src/encodings.rs b/crates/common/src/encodings.rs index 39ca2661262..d54581eb9ea 100644 --- a/crates/common/src/encodings.rs +++ b/crates/common/src/encodings.rs @@ -1,4 +1,4 @@ -use std::ops::{self, Range}; +use core::ops::{self, Range}; use num_traits::ToPrimitive; @@ -260,7 +260,7 @@ pub mod errors { use crate::str::UnicodeEscapeCodepoint; use super::*; - use std::fmt::Write; + use core::fmt::Write; pub struct Strict; diff --git a/crates/common/src/fileutils.rs b/crates/common/src/fileutils.rs index a12c1cd82e5..9ed5e77afbb 100644 --- a/crates/common/src/fileutils.rs +++ b/crates/common/src/fileutils.rs @@ -9,7 +9,7 @@ pub use windows::{StatStruct, fstat}; #[cfg(not(windows))] pub fn fstat(fd: crate::crt_fd::Borrowed<'_>) -> std::io::Result { - let mut stat = std::mem::MaybeUninit::uninit(); + let mut stat = core::mem::MaybeUninit::uninit(); unsafe { let ret = libc::fstat(fd.as_raw(), stat.as_mut_ptr()); if ret == -1 { @@ -165,7 +165,7 @@ pub mod windows { } fn file_time_to_time_t_nsec(in_ptr: &FILETIME) -> (libc::time_t, libc::c_int) { - let in_val: i64 = unsafe { std::mem::transmute_copy(in_ptr) }; + let in_val: i64 = unsafe { core::mem::transmute_copy(in_ptr) }; let nsec_out = (in_val % 10_000_000) * 100; // FILETIME is in units of 100 nsec. let time_out = (in_val / 10_000_000) - SECS_BETWEEN_EPOCHS; (time_out, nsec_out as _) @@ -204,7 +204,7 @@ pub mod windows { let st_nlink = info.nNumberOfLinks as i32; let st_ino = if let Some(id_info) = id_info { - let file_id: [u64; 2] = unsafe { std::mem::transmute_copy(&id_info.FileId) }; + let file_id: [u64; 2] = unsafe { core::mem::transmute_copy(&id_info.FileId) }; file_id } else { let ino = ((info.nFileIndexHigh as u64) << 32) + info.nFileIndexLow as u64; @@ -313,7 +313,7 @@ pub mod windows { unsafe { GetProcAddress(module, name.as_bytes_with_nul().as_ptr()) } { Some(unsafe { - std::mem::transmute::< + core::mem::transmute::< unsafe extern "system" fn() -> isize, unsafe extern "system" fn( *const u16, @@ -441,7 +441,7 @@ pub mod windows { // Open a file using std::fs::File and convert to FILE* // Automatically handles path encoding and EINTR retries pub fn fopen(path: &std::path::Path, mode: &str) -> std::io::Result<*mut libc::FILE> { - use std::ffi::CString; + use alloc::ffi::CString; use std::fs::File; // Currently only supports read mode diff --git a/crates/common/src/format.rs b/crates/common/src/format.rs index 447ae575f48..1afee519aef 100644 --- a/crates/common/src/format.rs +++ b/crates/common/src/format.rs @@ -1,4 +1,6 @@ // spell-checker:ignore ddfe +use core::ops::Deref; +use core::{cmp, str::FromStr}; use itertools::{Itertools, PeekingNext}; use malachite_base::num::basic::floats::PrimitiveFloat; use malachite_bigint::{BigInt, Sign}; @@ -7,8 +9,6 @@ use num_traits::FromPrimitive; use num_traits::{Signed, cast::ToPrimitive}; use rustpython_literal::float; use rustpython_literal::format::Case; -use std::ops::Deref; -use std::{cmp, str::FromStr}; use crate::wtf8::{CodePoint, Wtf8, Wtf8Buf}; @@ -598,7 +598,7 @@ impl FormatSpec { (Some(_), _) => Err(FormatSpecError::NotAllowed("Sign")), (_, true) => Err(FormatSpecError::NotAllowed("Alternate form (#)")), (_, _) => match num.to_u32() { - Some(n) if n <= 0x10ffff => Ok(std::char::from_u32(n).unwrap().to_string()), + Some(n) if n <= 0x10ffff => Ok(core::char::from_u32(n).unwrap().to_string()), Some(_) | None => Err(FormatSpecError::CodeNotInRange), }, }, diff --git a/crates/common/src/hash.rs b/crates/common/src/hash.rs index dcf424f7ba9..40c428d89e3 100644 --- a/crates/common/src/hash.rs +++ b/crates/common/src/hash.rs @@ -1,7 +1,7 @@ +use core::hash::{BuildHasher, Hash, Hasher}; use malachite_bigint::BigInt; use num_traits::ToPrimitive; use siphasher::sip::SipHasher24; -use std::hash::{BuildHasher, Hash, Hasher}; pub type PyHash = i64; pub type PyUHash = u64; @@ -19,9 +19,9 @@ pub const INF: PyHash = 314_159; pub const NAN: PyHash = 0; pub const IMAG: PyHash = MULTIPLIER; pub const ALGO: &str = "siphash24"; -pub const HASH_BITS: usize = std::mem::size_of::() * 8; +pub const HASH_BITS: usize = core::mem::size_of::() * 8; // SipHasher24 takes 2 u64s as a seed -pub const SEED_BITS: usize = std::mem::size_of::() * 2 * 8; +pub const SEED_BITS: usize = core::mem::size_of::() * 2 * 8; // pub const CUTOFF: usize = 7; @@ -134,7 +134,7 @@ pub fn hash_bigint(value: &BigInt) -> PyHash { Some(i) => mod_int(i), None => (value % MODULUS).to_i64().unwrap_or_else(|| unsafe { // SAFETY: MODULUS < i64::MAX, so value % MODULUS is guaranteed to be in the range of i64 - std::hint::unreachable_unchecked() + core::hint::unreachable_unchecked() }), }; fix_sentinel(ret) diff --git a/crates/common/src/int.rs b/crates/common/src/int.rs index ed09cc01a0a..57696e21fe7 100644 --- a/crates/common/src/int.rs +++ b/crates/common/src/int.rs @@ -7,18 +7,18 @@ pub fn true_div(numerator: &BigInt, denominator: &BigInt) -> f64 { let rational = Rational::from_integers_ref(numerator.into(), denominator.into()); match rational.rounding_into(RoundingMode::Nearest) { // returned value is $t::MAX but still less than the original - (val, std::cmp::Ordering::Less) if val == f64::MAX => f64::INFINITY, + (val, core::cmp::Ordering::Less) if val == f64::MAX => f64::INFINITY, // returned value is $t::MIN but still greater than the original - (val, std::cmp::Ordering::Greater) if val == f64::MIN => f64::NEG_INFINITY, + (val, core::cmp::Ordering::Greater) if val == f64::MIN => f64::NEG_INFINITY, (val, _) => val, } } pub fn float_to_ratio(value: f64) -> Option<(BigInt, BigInt)> { - let sign = match std::cmp::PartialOrd::partial_cmp(&value, &0.0)? { - std::cmp::Ordering::Less => Sign::Minus, - std::cmp::Ordering::Equal => return Some((BigInt::zero(), BigInt::one())), - std::cmp::Ordering::Greater => Sign::Plus, + let sign = match core::cmp::PartialOrd::partial_cmp(&value, &0.0)? { + core::cmp::Ordering::Less => Sign::Minus, + core::cmp::Ordering::Equal => return Some((BigInt::zero(), BigInt::one())), + core::cmp::Ordering::Greater => Sign::Plus, }; Rational::try_from(value).ok().map(|x| { let (numer, denom) = x.into_numerator_and_denominator(); diff --git a/crates/common/src/lib.rs b/crates/common/src/lib.rs index c99ba0286a4..0181562d043 100644 --- a/crates/common/src/lib.rs +++ b/crates/common/src/lib.rs @@ -2,6 +2,8 @@ #![cfg_attr(all(target_os = "wasi", target_env = "p2"), feature(wasip2))] +extern crate alloc; + #[macro_use] mod macros; pub use macros::*; diff --git a/crates/common/src/linked_list.rs b/crates/common/src/linked_list.rs index 8afc1478e6b..fb2b1260346 100644 --- a/crates/common/src/linked_list.rs +++ b/crates/common/src/linked_list.rs @@ -253,7 +253,7 @@ impl LinkedList { // === rustpython additions === pub fn iter(&self) -> impl Iterator { - std::iter::successors(self.head, |node| unsafe { + core::iter::successors(self.head, |node| unsafe { L::pointers(*node).as_ref().get_next() }) .map(|ptr| unsafe { ptr.as_ref() }) diff --git a/crates/common/src/lock/cell_lock.rs b/crates/common/src/lock/cell_lock.rs index 25a5cfedba1..73d722a8fdb 100644 --- a/crates/common/src/lock/cell_lock.rs +++ b/crates/common/src/lock/cell_lock.rs @@ -1,9 +1,9 @@ // spell-checker:ignore upgradably sharedly +use core::{cell::Cell, num::NonZero}; use lock_api::{ GetThreadId, RawMutex, RawRwLock, RawRwLockDowngrade, RawRwLockRecursive, RawRwLockUpgrade, RawRwLockUpgradeDowngrade, }; -use std::{cell::Cell, num::NonZero}; pub struct RawCellMutex { locked: Cell, diff --git a/crates/common/src/lock/immutable_mutex.rs b/crates/common/src/lock/immutable_mutex.rs index 81c5c93be71..2013cf1c60d 100644 --- a/crates/common/src/lock/immutable_mutex.rs +++ b/crates/common/src/lock/immutable_mutex.rs @@ -1,7 +1,8 @@ #![allow(clippy::needless_lifetimes)] +use alloc::fmt; +use core::{marker::PhantomData, ops::Deref}; use lock_api::{MutexGuard, RawMutex}; -use std::{fmt, marker::PhantomData, ops::Deref}; /// A mutex guard that has an exclusive lock, but only an immutable reference; useful if you /// need to map a mutex guard with a function that returns an `&T`. Construct using the @@ -22,7 +23,7 @@ impl<'a, R: RawMutex, T: ?Sized> MapImmutable<'a, R, T> for MutexGuard<'a, R, T> { let raw = unsafe { MutexGuard::mutex(&s).raw() }; let data = f(&s) as *const U; - std::mem::forget(s); + core::mem::forget(s); ImmutableMappedMutexGuard { raw, data, @@ -38,7 +39,7 @@ impl<'a, R: RawMutex, T: ?Sized> ImmutableMappedMutexGuard<'a, R, T> { { let raw = s.raw; let data = f(&s) as *const U; - std::mem::forget(s); + core::mem::forget(s); ImmutableMappedMutexGuard { raw, data, diff --git a/crates/common/src/lock/thread_mutex.rs b/crates/common/src/lock/thread_mutex.rs index 2949a3c6c14..67ffc89245d 100644 --- a/crates/common/src/lock/thread_mutex.rs +++ b/crates/common/src/lock/thread_mutex.rs @@ -1,14 +1,14 @@ #![allow(clippy::needless_lifetimes)] -use lock_api::{GetThreadId, GuardNoSend, RawMutex}; -use std::{ +use alloc::fmt; +use core::{ cell::UnsafeCell, - fmt, marker::PhantomData, ops::{Deref, DerefMut}, ptr::NonNull, sync::atomic::{AtomicUsize, Ordering}, }; +use lock_api::{GetThreadId, GuardNoSend, RawMutex}; // based off ReentrantMutex from lock_api @@ -174,7 +174,7 @@ impl<'a, R: RawMutex, G: GetThreadId, T: ?Sized> ThreadMutexGuard<'a, R, G, T> { ) -> MappedThreadMutexGuard<'a, R, G, U> { let data = f(&mut s).into(); let mu = &s.mu.raw; - std::mem::forget(s); + core::mem::forget(s); MappedThreadMutexGuard { mu, data, @@ -188,7 +188,7 @@ impl<'a, R: RawMutex, G: GetThreadId, T: ?Sized> ThreadMutexGuard<'a, R, G, T> { if let Some(data) = f(&mut s) { let data = data.into(); let mu = &s.mu.raw; - std::mem::forget(s); + core::mem::forget(s); Ok(MappedThreadMutexGuard { mu, data, @@ -241,7 +241,7 @@ impl<'a, R: RawMutex, G: GetThreadId, T: ?Sized> MappedThreadMutexGuard<'a, R, G ) -> MappedThreadMutexGuard<'a, R, G, U> { let data = f(&mut s).into(); let mu = s.mu; - std::mem::forget(s); + core::mem::forget(s); MappedThreadMutexGuard { mu, data, @@ -255,7 +255,7 @@ impl<'a, R: RawMutex, G: GetThreadId, T: ?Sized> MappedThreadMutexGuard<'a, R, G if let Some(data) = f(&mut s) { let data = data.into(); let mu = s.mu; - std::mem::forget(s); + core::mem::forget(s); Ok(MappedThreadMutexGuard { mu, data, diff --git a/crates/common/src/os.rs b/crates/common/src/os.rs index e77a81fd94f..3e09a29210a 100644 --- a/crates/common/src/os.rs +++ b/crates/common/src/os.rs @@ -1,7 +1,8 @@ // spell-checker:disable // TODO: we can move more os-specific bindings/interfaces from stdlib::{os, posix, nt} to here -use std::{io, process::ExitCode, str::Utf8Error}; +use core::str::Utf8Error; +use std::{io, process::ExitCode}; /// Convert exit code to std::process::ExitCode /// diff --git a/crates/common/src/rc.rs b/crates/common/src/rc.rs index 40c7cf97a8d..9e4cca228fd 100644 --- a/crates/common/src/rc.rs +++ b/crates/common/src/rc.rs @@ -1,7 +1,7 @@ #[cfg(not(feature = "threading"))] -use std::rc::Rc; +use alloc::rc::Rc; #[cfg(feature = "threading")] -use std::sync::Arc; +use alloc::sync::Arc; // type aliases instead of new-types because you can't do `fn method(self: PyRc)` with a // newtype; requires the arbitrary_self_types unstable feature diff --git a/crates/common/src/str.rs b/crates/common/src/str.rs index 2d867130edd..155012ed21f 100644 --- a/crates/common/src/str.rs +++ b/crates/common/src/str.rs @@ -4,8 +4,8 @@ use crate::format::CharLen; use crate::wtf8::{CodePoint, Wtf8, Wtf8Buf}; use ascii::{AsciiChar, AsciiStr, AsciiString}; use core::fmt; +use core::ops::{Bound, RangeBounds}; use core::sync::atomic::Ordering::Relaxed; -use std::ops::{Bound, RangeBounds}; #[cfg(not(target_arch = "wasm32"))] #[allow(non_camel_case_types)] @@ -22,7 +22,7 @@ pub enum StrKind { Wtf8, } -impl std::ops::BitOr for StrKind { +impl core::ops::BitOr for StrKind { type Output = Self; fn bitor(self, other: Self) -> Self { @@ -128,7 +128,7 @@ impl From for StrLen { } impl fmt::Debug for StrLen { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result { let len = self.0.load(Relaxed); if len == usize::MAX { f.write_str("") @@ -262,7 +262,7 @@ impl StrData { pub fn as_str(&self) -> Option<&str> { self.kind .is_utf8() - .then(|| unsafe { std::str::from_utf8_unchecked(self.data.as_bytes()) }) + .then(|| unsafe { core::str::from_utf8_unchecked(self.data.as_bytes()) }) } pub fn as_ascii(&self) -> Option<&AsciiStr> { @@ -282,7 +282,7 @@ impl StrData { PyKindStr::Ascii(unsafe { AsciiStr::from_ascii_unchecked(self.data.as_bytes()) }) } StrKind::Utf8 => { - PyKindStr::Utf8(unsafe { std::str::from_utf8_unchecked(self.data.as_bytes()) }) + PyKindStr::Utf8(unsafe { core::str::from_utf8_unchecked(self.data.as_bytes()) }) } StrKind::Wtf8 => PyKindStr::Wtf8(&self.data), } @@ -327,8 +327,8 @@ impl StrData { } } -impl std::fmt::Display for StrData { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { +impl core::fmt::Display for StrData { + fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result { self.data.fmt(f) } } @@ -421,7 +421,7 @@ pub fn zfill(bytes: &[u8], width: usize) -> Vec { }; let mut filled = Vec::new(); filled.extend_from_slice(sign); - filled.extend(std::iter::repeat_n(b'0', width - bytes.len())); + filled.extend(core::iter::repeat_n(b'0', width - bytes.len())); filled.extend_from_slice(s); filled } @@ -465,7 +465,8 @@ impl fmt::Display for UnicodeEscapeCodepoint { } pub mod levenshtein { - use std::{cell::RefCell, thread_local}; + use core::cell::RefCell; + use std::thread_local; pub const MOVE_COST: usize = 2; const CASE_COST: usize = 1; @@ -524,9 +525,9 @@ pub mod levenshtein { } if b_end < a_end { - std::mem::swap(&mut a_bytes, &mut b_bytes); - std::mem::swap(&mut a_begin, &mut b_begin); - std::mem::swap(&mut a_end, &mut b_end); + core::mem::swap(&mut a_bytes, &mut b_bytes); + core::mem::swap(&mut a_begin, &mut b_begin); + core::mem::swap(&mut a_end, &mut b_end); } if (b_end - a_end) * MOVE_COST > max_cost { diff --git a/crates/compiler-core/src/bytecode.rs b/crates/compiler-core/src/bytecode.rs index 8df5d9caf6f..5569fa2012b 100644 --- a/crates/compiler-core/src/bytecode.rs +++ b/crates/compiler-core/src/bytecode.rs @@ -5,12 +5,13 @@ use crate::{ marshal::MarshalError, {OneIndexed, SourceLocation}, }; +use alloc::{collections::BTreeSet, fmt}; use bitflags::bitflags; +use core::{hash, marker::PhantomData, mem, num::NonZeroU8, ops::Deref}; use itertools::Itertools; use malachite_bigint::BigInt; use num_complex::Complex64; use rustpython_wtf8::{Wtf8, Wtf8Buf}; -use std::{collections::BTreeSet, fmt, hash, marker::PhantomData, mem, num::NonZeroU8, ops::Deref}; /// Oparg values for [`Instruction::ConvertValue`]. /// @@ -506,7 +507,7 @@ impl Eq for Arg {} impl fmt::Debug for Arg { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - write!(f, "Arg<{}>", std::any::type_name::()) + write!(f, "Arg<{}>", core::any::type_name::()) } } @@ -880,7 +881,7 @@ impl From for u8 { #[inline] fn from(ins: Instruction) -> Self { // SAFETY: there's no padding bits - unsafe { std::mem::transmute::(ins) } + unsafe { core::mem::transmute::(ins) } } } @@ -890,7 +891,7 @@ impl TryFrom for Instruction { #[inline] fn try_from(value: u8) -> Result { if value <= u8::from(LAST_INSTRUCTION) { - Ok(unsafe { std::mem::transmute::(value) }) + Ok(unsafe { core::mem::transmute::(value) }) } else { Err(MarshalError::InvalidBytecode) } @@ -1027,7 +1028,7 @@ impl PartialEq for ConstantData { (Boolean { value: a }, Boolean { value: b }) => a == b, (Str { value: a }, Str { value: b }) => a == b, (Bytes { value: a }, Bytes { value: b }) => a == b, - (Code { code: a }, Code { code: b }) => std::ptr::eq(a.as_ref(), b.as_ref()), + (Code { code: a }, Code { code: b }) => core::ptr::eq(a.as_ref(), b.as_ref()), (Tuple { elements: a }, Tuple { elements: b }) => a == b, (None, None) => true, (Ellipsis, Ellipsis) => true, @@ -1053,7 +1054,7 @@ impl hash::Hash for ConstantData { Boolean { value } => value.hash(state), Str { value } => value.hash(state), Bytes { value } => value.hash(state), - Code { code } => std::ptr::hash(code.as_ref(), state), + Code { code } => core::ptr::hash(code.as_ref(), state), Tuple { elements } => elements.hash(state), None => {} Ellipsis => {} diff --git a/crates/compiler-core/src/lib.rs b/crates/compiler-core/src/lib.rs index 08cdc0ec21f..11246f6f44c 100644 --- a/crates/compiler-core/src/lib.rs +++ b/crates/compiler-core/src/lib.rs @@ -1,6 +1,8 @@ #![doc(html_logo_url = "https://raw.githubusercontent.com/RustPython/RustPython/main/logo.png")] #![doc(html_root_url = "https://docs.rs/rustpython-compiler-core/")] +extern crate alloc; + pub mod bytecode; pub mod frozen; pub mod marshal; diff --git a/crates/compiler-core/src/marshal.rs b/crates/compiler-core/src/marshal.rs index 39e48071678..b30894ea065 100644 --- a/crates/compiler-core/src/marshal.rs +++ b/crates/compiler-core/src/marshal.rs @@ -1,8 +1,8 @@ use crate::{OneIndexed, SourceLocation, bytecode::*}; +use core::convert::Infallible; use malachite_bigint::{BigInt, Sign}; use num_complex::Complex64; use rustpython_wtf8::Wtf8; -use std::convert::Infallible; pub const FORMAT_VERSION: u32 = 4; @@ -20,8 +20,8 @@ pub enum MarshalError { BadType, } -impl std::fmt::Display for MarshalError { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { +impl core::fmt::Display for MarshalError { + fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result { match self { Self::Eof => f.write_str("unexpected end of data"), Self::InvalidBytecode => f.write_str("invalid bytecode"), @@ -32,15 +32,15 @@ impl std::fmt::Display for MarshalError { } } -impl From for MarshalError { - fn from(_: std::str::Utf8Error) -> Self { +impl From for MarshalError { + fn from(_: core::str::Utf8Error) -> Self { Self::InvalidUtf8 } } -impl std::error::Error for MarshalError {} +impl core::error::Error for MarshalError {} -type Result = std::result::Result; +type Result = core::result::Result; #[repr(u8)] enum Type { @@ -119,7 +119,7 @@ pub trait Read { } fn read_str(&mut self, len: u32) -> Result<&str> { - Ok(std::str::from_utf8(self.read_slice(len)?)?) + Ok(core::str::from_utf8(self.read_slice(len)?)?) } fn read_wtf8(&mut self, len: u32) -> Result<&Wtf8> { @@ -147,7 +147,7 @@ pub(crate) trait ReadBorrowed<'a>: Read { fn read_slice_borrow(&mut self, n: u32) -> Result<&'a [u8]>; fn read_str_borrow(&mut self, len: u32) -> Result<&'a str> { - Ok(std::str::from_utf8(self.read_slice_borrow(len)?)?) + Ok(core::str::from_utf8(self.read_slice_borrow(len)?)?) } } diff --git a/crates/compiler-core/src/mode.rs b/crates/compiler-core/src/mode.rs index 35e9e77f590..f2b19d677be 100644 --- a/crates/compiler-core/src/mode.rs +++ b/crates/compiler-core/src/mode.rs @@ -7,7 +7,7 @@ pub enum Mode { BlockExpr, } -impl std::str::FromStr for Mode { +impl core::str::FromStr for Mode { type Err = ModeParseError; // To support `builtins.compile()` `mode` argument @@ -25,8 +25,8 @@ impl std::str::FromStr for Mode { #[derive(Debug)] pub struct ModeParseError; -impl std::fmt::Display for ModeParseError { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { +impl core::fmt::Display for ModeParseError { + fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result { write!(f, r#"mode must be "exec", "eval", or "single""#) } } diff --git a/crates/compiler/src/lib.rs b/crates/compiler/src/lib.rs index 84e64f3c27f..7fa695c0c71 100644 --- a/crates/compiler/src/lib.rs +++ b/crates/compiler/src/lib.rs @@ -28,8 +28,8 @@ pub struct ParseError { pub source_path: String, } -impl std::fmt::Display for ParseError { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { +impl ::core::fmt::Display for ParseError { + fn fmt(&self, f: &mut ::core::fmt::Formatter<'_>) -> ::core::fmt::Result { self.error.fmt(f) } } diff --git a/crates/derive-impl/src/compile_bytecode.rs b/crates/derive-impl/src/compile_bytecode.rs index cdcc89b9984..23c90690dad 100644 --- a/crates/derive-impl/src/compile_bytecode.rs +++ b/crates/derive-impl/src/compile_bytecode.rs @@ -58,11 +58,11 @@ pub trait Compiler { source: &str, mode: Mode, module_name: String, - ) -> Result>; + ) -> Result>; } impl CompilationSource { - fn compile_string D>( + fn compile_string D>( &self, source: &str, mode: Mode, diff --git a/crates/derive-impl/src/from_args.rs b/crates/derive-impl/src/from_args.rs index 4633c9b3aac..667f887e81c 100644 --- a/crates/derive-impl/src/from_args.rs +++ b/crates/derive-impl/src/from_args.rs @@ -18,7 +18,7 @@ enum ParameterKind { impl TryFrom<&Ident> for ParameterKind { type Error = (); - fn try_from(ident: &Ident) -> std::result::Result { + fn try_from(ident: &Ident) -> core::result::Result { Ok(match ident.to_string().as_str() { "positional" => Self::PositionalOnly, "any" => Self::PositionalOrKeyword, @@ -105,12 +105,12 @@ impl ArgAttribute { impl TryFrom<&Field> for ArgAttribute { type Error = syn::Error; - fn try_from(field: &Field) -> std::result::Result { + fn try_from(field: &Field) -> core::result::Result { let mut pyarg_attrs = field .attrs .iter() .filter_map(Self::from_attribute) - .collect::, _>>()?; + .collect::, _>>()?; if pyarg_attrs.len() >= 2 { bail_span!(field, "Multiple pyarg attributes on field") @@ -234,7 +234,7 @@ pub fn impl_from_args(input: DeriveInput) -> Result { fn from_args( vm: &::rustpython_vm::VirtualMachine, args: &mut ::rustpython_vm::function::FuncArgs - ) -> ::std::result::Result { + ) -> ::core::result::Result { Ok(Self { #fields }) } } diff --git a/crates/derive-impl/src/pyclass.rs b/crates/derive-impl/src/pyclass.rs index 55f9c769940..06bbc06cfb2 100644 --- a/crates/derive-impl/src/pyclass.rs +++ b/crates/derive-impl/src/pyclass.rs @@ -4,11 +4,11 @@ use crate::util::{ ItemMeta, ItemMetaInner, ItemNursery, SimpleItemMeta, format_doc, pyclass_ident_and_attrs, pyexception_ident_and_attrs, text_signature, }; +use core::str::FromStr; use proc_macro2::{Delimiter, Group, Span, TokenStream, TokenTree}; use quote::{ToTokens, quote, quote_spanned}; use rustpython_doc::DB; use std::collections::{HashMap, HashSet}; -use std::str::FromStr; use syn::{Attribute, Ident, Item, Result, parse_quote, spanned::Spanned}; use syn_ext::ext::*; use syn_ext::types::*; @@ -25,8 +25,8 @@ enum AttrName { Member, } -impl std::fmt::Display for AttrName { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { +impl core::fmt::Display for AttrName { + fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result { let s = match self { Self::Method => "pymethod", Self::ClassMethod => "pyclassmethod", @@ -44,7 +44,7 @@ impl std::fmt::Display for AttrName { impl FromStr for AttrName { type Err = String; - fn from_str(s: &str) -> std::result::Result { + fn from_str(s: &str) -> core::result::Result { Ok(match s { "pymethod" => Self::Method, "pyclassmethod" => Self::ClassMethod, @@ -1488,7 +1488,7 @@ impl ItemMeta for SlotItemMeta { fn from_nested(item_ident: Ident, meta_ident: Ident, mut nested: I) -> Result where - I: std::iter::Iterator, + I: core::iter::Iterator, { let meta_map = if let Some(nested_meta) = nested.next() { match nested_meta { diff --git a/crates/derive-impl/src/pymodule.rs b/crates/derive-impl/src/pymodule.rs index 2d5ff7cb0c2..3689ac97fd8 100644 --- a/crates/derive-impl/src/pymodule.rs +++ b/crates/derive-impl/src/pymodule.rs @@ -5,10 +5,11 @@ use crate::util::{ ErrorVec, ItemMeta, ItemNursery, ModuleItemMeta, SimpleItemMeta, format_doc, iter_use_idents, pyclass_ident_and_attrs, text_signature, }; +use core::str::FromStr; use proc_macro2::{Delimiter, Group, TokenStream, TokenTree}; use quote::{ToTokens, quote, quote_spanned}; use rustpython_doc::DB; -use std::{collections::HashSet, str::FromStr}; +use std::collections::HashSet; use syn::{Attribute, Ident, Item, Result, parse_quote, spanned::Spanned}; use syn_ext::ext::*; use syn_ext::types::PunctuatedNestedMeta; @@ -22,8 +23,8 @@ enum AttrName { StructSequence, } -impl std::fmt::Display for AttrName { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { +impl core::fmt::Display for AttrName { + fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result { let s = match self { Self::Function => "pyfunction", Self::Attr => "pyattr", @@ -38,7 +39,7 @@ impl std::fmt::Display for AttrName { impl FromStr for AttrName { type Err = String; - fn from_str(s: &str) -> std::result::Result { + fn from_str(s: &str) -> core::result::Result { Ok(match s { "pyfunction" => Self::Function, "pyattr" => Self::Attr, diff --git a/crates/derive-impl/src/pytraverse.rs b/crates/derive-impl/src/pytraverse.rs index c5c4bbd2704..c4ec3823298 100644 --- a/crates/derive-impl/src/pytraverse.rs +++ b/crates/derive-impl/src/pytraverse.rs @@ -37,7 +37,7 @@ fn field_to_traverse_code(field: &Field) -> Result { .attrs .iter() .filter_map(pytraverse_arg) - .collect::, _>>()?; + .collect::, _>>()?; let do_trace = if pytraverse_attrs.len() > 1 { bail_span!( field, diff --git a/crates/derive-impl/src/util.rs b/crates/derive-impl/src/util.rs index 379adc65b57..6be1fcdf7ad 100644 --- a/crates/derive-impl/src/util.rs +++ b/crates/derive-impl/src/util.rs @@ -97,7 +97,7 @@ pub(crate) struct ContentItemInner { } pub(crate) trait ContentItem { - type AttrName: std::str::FromStr + std::fmt::Display; + type AttrName: core::str::FromStr + core::fmt::Display; fn inner(&self) -> &ContentItemInner; fn index(&self) -> usize { @@ -125,7 +125,7 @@ impl ItemMetaInner { allowed_names: &[&'static str], ) -> Result where - I: std::iter::Iterator, + I: core::iter::Iterator, { let (meta_map, lits) = nested.into_unique_map_and_lits(|path| { if let Some(ident) = path.get_ident() { @@ -243,7 +243,7 @@ impl ItemMetaInner { pub fn _optional_list( &self, key: &str, - ) -> Result>> { + ) -> Result>> { let value = if let Some((_, meta)) = self.meta_map.get(key) { let Meta::List(MetaList { path: _, nested, .. @@ -269,7 +269,7 @@ pub(crate) trait ItemMeta: Sized { fn from_nested(item_ident: Ident, meta_ident: Ident, nested: I) -> Result where - I: std::iter::Iterator, + I: core::iter::Iterator, { Ok(Self::from_inner(ItemMetaInner::from_nested( item_ident, @@ -529,7 +529,7 @@ impl ExceptionItemMeta { } } -impl std::ops::Deref for ExceptionItemMeta { +impl core::ops::Deref for ExceptionItemMeta { type Target = ClassItemMeta; fn deref(&self) -> &Self::Target { &self.0 diff --git a/crates/derive/src/lib.rs b/crates/derive/src/lib.rs index 655ad3b4c9e..5a3ff84c63a 100644 --- a/crates/derive/src/lib.rs +++ b/crates/derive/src/lib.rs @@ -274,7 +274,7 @@ impl derive_impl::Compiler for Compiler { source: &str, mode: rustpython_compiler::Mode, module_name: String, - ) -> Result> { + ) -> Result> { use rustpython_compiler::{CompileOpts, compile}; Ok(compile(source, mode, &module_name, CompileOpts::default())?) } diff --git a/crates/jit/src/lib.rs b/crates/jit/src/lib.rs index 65ef87a62f6..1e278617661 100644 --- a/crates/jit/src/lib.rs +++ b/crates/jit/src/lib.rs @@ -1,11 +1,14 @@ mod instructions; +extern crate alloc; + +use alloc::fmt; +use core::mem::ManuallyDrop; use cranelift::prelude::*; use cranelift_jit::{JITBuilder, JITModule}; use cranelift_module::{FuncId, Linkage, Module, ModuleError}; use instructions::FunctionCompiler; use rustpython_compiler_core::bytecode; -use std::{fmt, mem::ManuallyDrop}; #[derive(Debug, thiserror::Error)] #[non_exhaustive] diff --git a/crates/literal/src/escape.rs b/crates/literal/src/escape.rs index 6bdd94e9860..72ceaf60d5b 100644 --- a/crates/literal/src/escape.rs +++ b/crates/literal/src/escape.rs @@ -55,9 +55,9 @@ pub unsafe trait Escape { /// # Safety /// /// This string must only contain printable characters. - unsafe fn write_source(&self, formatter: &mut impl std::fmt::Write) -> std::fmt::Result; - fn write_body_slow(&self, formatter: &mut impl std::fmt::Write) -> std::fmt::Result; - fn write_body(&self, formatter: &mut impl std::fmt::Write) -> std::fmt::Result { + unsafe fn write_source(&self, formatter: &mut impl std::fmt::Write) -> core::fmt::Result; + fn write_body_slow(&self, formatter: &mut impl std::fmt::Write) -> core::fmt::Result; + fn write_body(&self, formatter: &mut impl std::fmt::Write) -> core::fmt::Result { if self.changed() { self.write_body_slow(formatter) } else { @@ -117,7 +117,7 @@ impl<'a> UnicodeEscape<'a> { pub struct StrRepr<'r, 'a>(&'r UnicodeEscape<'a>); impl StrRepr<'_, '_> { - pub fn write(&self, formatter: &mut impl std::fmt::Write) -> std::fmt::Result { + pub fn write(&self, formatter: &mut impl std::fmt::Write) -> core::fmt::Result { let quote = self.0.layout().quote.to_char(); formatter.write_char(quote)?; self.0.write_body(formatter)?; @@ -131,8 +131,8 @@ impl StrRepr<'_, '_> { } } -impl std::fmt::Display for StrRepr<'_, '_> { - fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { +impl core::fmt::Display for StrRepr<'_, '_> { + fn fmt(&self, formatter: &mut core::fmt::Formatter<'_>) -> core::fmt::Result { self.write(formatter) } } @@ -217,7 +217,7 @@ impl UnicodeEscape<'_> { ch: CodePoint, quote: Quote, formatter: &mut impl std::fmt::Write, - ) -> std::fmt::Result { + ) -> core::fmt::Result { let Some(ch) = ch.to_char() else { return write!(formatter, "\\u{:04x}", ch.to_u32()); }; @@ -260,7 +260,7 @@ unsafe impl Escape for UnicodeEscape<'_> { &self.layout } - unsafe fn write_source(&self, formatter: &mut impl std::fmt::Write) -> std::fmt::Result { + unsafe fn write_source(&self, formatter: &mut impl std::fmt::Write) -> core::fmt::Result { formatter.write_str(unsafe { // SAFETY: this function must be called only when source is printable characters (i.e. no surrogates) std::str::from_utf8_unchecked(self.source.as_bytes()) @@ -268,7 +268,7 @@ unsafe impl Escape for UnicodeEscape<'_> { } #[cold] - fn write_body_slow(&self, formatter: &mut impl std::fmt::Write) -> std::fmt::Result { + fn write_body_slow(&self, formatter: &mut impl std::fmt::Write) -> core::fmt::Result { for ch in self.source.code_points() { Self::write_char(ch, self.layout().quote, formatter)?; } @@ -378,7 +378,7 @@ impl AsciiEscape<'_> { } } - fn write_char(ch: u8, quote: Quote, formatter: &mut impl std::fmt::Write) -> std::fmt::Result { + fn write_char(ch: u8, quote: Quote, formatter: &mut impl std::fmt::Write) -> core::fmt::Result { match ch { b'\t' => formatter.write_str("\\t"), b'\n' => formatter.write_str("\\n"), @@ -404,7 +404,7 @@ unsafe impl Escape for AsciiEscape<'_> { &self.layout } - unsafe fn write_source(&self, formatter: &mut impl std::fmt::Write) -> std::fmt::Result { + unsafe fn write_source(&self, formatter: &mut impl std::fmt::Write) -> core::fmt::Result { formatter.write_str(unsafe { // SAFETY: this function must be called only when source is printable ascii characters std::str::from_utf8_unchecked(self.source) @@ -412,7 +412,7 @@ unsafe impl Escape for AsciiEscape<'_> { } #[cold] - fn write_body_slow(&self, formatter: &mut impl std::fmt::Write) -> std::fmt::Result { + fn write_body_slow(&self, formatter: &mut impl std::fmt::Write) -> core::fmt::Result { for ch in self.source { Self::write_char(*ch, self.layout().quote, formatter)?; } @@ -423,7 +423,7 @@ unsafe impl Escape for AsciiEscape<'_> { pub struct BytesRepr<'r, 'a>(&'r AsciiEscape<'a>); impl BytesRepr<'_, '_> { - pub fn write(&self, formatter: &mut impl std::fmt::Write) -> std::fmt::Result { + pub fn write(&self, formatter: &mut impl std::fmt::Write) -> core::fmt::Result { let quote = self.0.layout().quote.to_char(); formatter.write_char('b')?; formatter.write_char(quote)?; @@ -438,8 +438,8 @@ impl BytesRepr<'_, '_> { } } -impl std::fmt::Display for BytesRepr<'_, '_> { - fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { +impl core::fmt::Display for BytesRepr<'_, '_> { + fn fmt(&self, formatter: &mut core::fmt::Formatter<'_>) -> core::fmt::Result { self.write(formatter) } } diff --git a/crates/literal/src/float.rs b/crates/literal/src/float.rs index e2bc54a8f1b..4d0d65cbb34 100644 --- a/crates/literal/src/float.rs +++ b/crates/literal/src/float.rs @@ -55,7 +55,7 @@ pub fn format_fixed(precision: usize, magnitude: f64, case: Case, alternate_form match magnitude { magnitude if magnitude.is_finite() => { let point = decimal_point_or_empty(precision, alternate_form); - let precision = std::cmp::min(precision, u16::MAX as usize); + let precision = core::cmp::min(precision, u16::MAX as usize); format!("{magnitude:.precision$}{point}") } magnitude if magnitude.is_nan() => format_nan(case), diff --git a/crates/sre_engine/src/engine.rs b/crates/sre_engine/src/engine.rs index 9cc2e4788a5..f1f25a2d920 100644 --- a/crates/sre_engine/src/engine.rs +++ b/crates/sre_engine/src/engine.rs @@ -6,8 +6,8 @@ use crate::string::{ }; use super::{MAXREPEAT, SreAtCode, SreCatCode, SreInfo, SreOpcode, StrDrive, StringCursor}; +use core::{convert::TryFrom, ptr::null}; use optional::Optioned; -use std::{convert::TryFrom, ptr::null}; #[derive(Debug, Clone, Copy)] pub struct Request<'a, S> { @@ -27,8 +27,8 @@ impl<'a, S: StrDrive> Request<'a, S> { pattern_codes: &'a [u32], match_all: bool, ) -> Self { - let end = std::cmp::min(end, string.count()); - let start = std::cmp::min(start, end); + let end = core::cmp::min(end, string.count()); + let start = core::cmp::min(start, end); Self { string, @@ -1332,7 +1332,7 @@ fn _count( ctx: &mut MatchContext, max_count: usize, ) -> usize { - let max_count = std::cmp::min(max_count, ctx.remaining_chars(req)); + let max_count = core::cmp::min(max_count, ctx.remaining_chars(req)); let end = ctx.cursor.position + max_count; let opcode = SreOpcode::try_from(ctx.peek_code(req, 0)).unwrap(); diff --git a/crates/sre_engine/src/string.rs b/crates/sre_engine/src/string.rs index 0d3325b6a1d..489819bfb3e 100644 --- a/crates/sre_engine/src/string.rs +++ b/crates/sre_engine/src/string.rs @@ -9,7 +9,7 @@ pub struct StringCursor { impl Default for StringCursor { fn default() -> Self { Self { - ptr: std::ptr::null(), + ptr: core::ptr::null(), position: 0, } } diff --git a/crates/stdlib/src/array.rs b/crates/stdlib/src/array.rs index b51bc02d3fb..b7a6fbd8b4f 100644 --- a/crates/stdlib/src/array.rs +++ b/crates/stdlib/src/array.rs @@ -68,11 +68,12 @@ mod array { }, }, }; + use alloc::fmt; + use core::cmp::Ordering; use itertools::Itertools; use num_traits::ToPrimitive; use rustpython_common::wtf8::{CodePoint, Wtf8, Wtf8Buf}; - use std::{cmp::Ordering, fmt, os::raw}; - + use std::os::raw; macro_rules! def_array_enum { ($(($n:ident, $t:ty, $c:literal, $scode:literal)),*$(,)?) => { #[derive(Debug, Clone)] @@ -104,14 +105,14 @@ mod array { const fn itemsize_of_typecode(c: char) -> Option { match c { - $($c => Some(std::mem::size_of::<$t>()),)* + $($c => Some(core::mem::size_of::<$t>()),)* _ => None, } } const fn itemsize(&self) -> usize { match self { - $(ArrayContentType::$n(_) => std::mem::size_of::<$t>(),)* + $(ArrayContentType::$n(_) => core::mem::size_of::<$t>(),)* } } @@ -201,10 +202,10 @@ mod array { if v.is_empty() { // safe because every configuration of bytes for the types we // support are valid - let b = std::mem::ManuallyDrop::new(b); + let b = core::mem::ManuallyDrop::new(b); let ptr = b.as_ptr() as *mut $t; - let len = b.len() / std::mem::size_of::<$t>(); - let capacity = b.capacity() / std::mem::size_of::<$t>(); + let len = b.len() / core::mem::size_of::<$t>(); + let capacity = b.capacity() / core::mem::size_of::<$t>(); *v = unsafe { Vec::from_raw_parts(ptr, len, capacity) }; } else { self.frombytes(&b); @@ -220,8 +221,8 @@ mod array { // support are valid if b.len() > 0 { let ptr = b.as_ptr() as *const $t; - let ptr_len = b.len() / std::mem::size_of::<$t>(); - let slice = unsafe { std::slice::from_raw_parts(ptr, ptr_len) }; + let ptr_len = b.len() / core::mem::size_of::<$t>(); + let slice = unsafe { core::slice::from_raw_parts(ptr, ptr_len) }; v.extend_from_slice(slice); } })* @@ -249,8 +250,8 @@ mod array { $(ArrayContentType::$n(v) => { // safe because we're just reading memory as bytes let ptr = v.as_ptr() as *const u8; - let ptr_len = v.len() * std::mem::size_of::<$t>(); - unsafe { std::slice::from_raw_parts(ptr, ptr_len) } + let ptr_len = v.len() * core::mem::size_of::<$t>(); + unsafe { core::slice::from_raw_parts(ptr, ptr_len) } })* } } @@ -260,8 +261,8 @@ mod array { $(ArrayContentType::$n(v) => { // safe because we're just reading memory as bytes let ptr = v.as_ptr() as *mut u8; - let ptr_len = v.len() * std::mem::size_of::<$t>(); - unsafe { std::slice::from_raw_parts_mut(ptr, ptr_len) } + let ptr_len = v.len() * core::mem::size_of::<$t>(); + unsafe { core::slice::from_raw_parts_mut(ptr, ptr_len) } })* } } @@ -785,18 +786,18 @@ mod array { if item_size == 2 { // safe because every configuration of bytes for the types we support are valid let utf16 = unsafe { - std::slice::from_raw_parts( + core::slice::from_raw_parts( bytes.as_ptr() as *const u16, - bytes.len() / std::mem::size_of::(), + bytes.len() / core::mem::size_of::(), ) }; Ok(Wtf8Buf::from_wide(utf16)) } else { // safe because every configuration of bytes for the types we support are valid let chars = unsafe { - std::slice::from_raw_parts( + core::slice::from_raw_parts( bytes.as_ptr() as *const u32, - bytes.len() / std::mem::size_of::(), + bytes.len() / core::mem::size_of::(), ) }; chars @@ -1516,7 +1517,7 @@ mod array { impl MachineFormatCode { fn from_typecode(code: char) -> Option { - use std::mem::size_of; + use core::mem::size_of; let signed = code.is_ascii_uppercase(); let big_endian = cfg!(target_endian = "big"); let int_size = match code { @@ -1590,7 +1591,7 @@ mod array { macro_rules! chunk_to_obj { ($BYTE:ident, $TY:ty, $BIG_ENDIAN:ident) => {{ - let b = <[u8; ::std::mem::size_of::<$TY>()]>::try_from($BYTE).unwrap(); + let b = <[u8; ::core::mem::size_of::<$TY>()]>::try_from($BYTE).unwrap(); if $BIG_ENDIAN { <$TY>::from_be_bytes(b) } else { @@ -1601,7 +1602,7 @@ mod array { chunk_to_obj!($BYTE, $TY, $BIG_ENDIAN).to_pyobject($VM) }; ($VM:ident, $BYTE:ident, $SIGNED_TY:ty, $UNSIGNED_TY:ty, $SIGNED:ident, $BIG_ENDIAN:ident) => {{ - let b = <[u8; ::std::mem::size_of::<$SIGNED_TY>()]>::try_from($BYTE).unwrap(); + let b = <[u8; ::core::mem::size_of::<$SIGNED_TY>()]>::try_from($BYTE).unwrap(); match ($SIGNED, $BIG_ENDIAN) { (false, false) => <$UNSIGNED_TY>::from_le_bytes(b).to_pyobject($VM), (false, true) => <$UNSIGNED_TY>::from_be_bytes(b).to_pyobject($VM), diff --git a/crates/stdlib/src/binascii.rs b/crates/stdlib/src/binascii.rs index a2316d3c204..671d1d9e253 100644 --- a/crates/stdlib/src/binascii.rs +++ b/crates/stdlib/src/binascii.rs @@ -359,7 +359,7 @@ mod decl { } _ => unsafe { // quad_pos is only assigned in this match statement to constants - std::hint::unreachable_unchecked() + core::hint::unreachable_unchecked() }, } } diff --git a/crates/stdlib/src/bz2.rs b/crates/stdlib/src/bz2.rs index a2a40953cff..93142e92a68 100644 --- a/crates/stdlib/src/bz2.rs +++ b/crates/stdlib/src/bz2.rs @@ -15,9 +15,10 @@ mod _bz2 { object::PyResult, types::Constructor, }; + use alloc::fmt; use bzip2::{Decompress, Status, write::BzEncoder}; use rustpython_vm::convert::ToPyException; - use std::{fmt, io::Write}; + use std::io::Write; const BUFSIZ: usize = 8192; diff --git a/crates/stdlib/src/cmath.rs b/crates/stdlib/src/cmath.rs index e5d1d55a578..7f975e41719 100644 --- a/crates/stdlib/src/cmath.rs +++ b/crates/stdlib/src/cmath.rs @@ -11,7 +11,7 @@ mod cmath { // Constants #[pyattr] - use std::f64::consts::{E as e, PI as pi, TAU as tau}; + use core::f64::consts::{E as e, PI as pi, TAU as tau}; #[pyattr(name = "inf")] const INF: f64 = f64::INFINITY; #[pyattr(name = "nan")] @@ -93,7 +93,7 @@ mod cmath { z.log( base.into_option() .map(|base| base.re) - .unwrap_or(std::f64::consts::E), + .unwrap_or(core::f64::consts::E), ) } diff --git a/crates/stdlib/src/compression.rs b/crates/stdlib/src/compression.rs index 7f4e3432eab..a857b4e53de 100644 --- a/crates/stdlib/src/compression.rs +++ b/crates/stdlib/src/compression.rs @@ -107,7 +107,7 @@ impl<'a> Chunker<'a> { pub fn advance(&mut self, consumed: usize) { self.data1 = &self.data1[consumed..]; if self.data1.is_empty() { - self.data1 = std::mem::take(&mut self.data2); + self.data1 = core::mem::take(&mut self.data2); } } } @@ -140,7 +140,7 @@ pub fn _decompress_chunks( let chunk = data.chunk(); let flush = calc_flush(chunk.len() == data.len()); loop { - let additional = std::cmp::min(bufsize, max_length - buf.capacity()); + let additional = core::cmp::min(bufsize, max_length - buf.capacity()); if additional == 0 { return Ok((buf, false)); } diff --git a/crates/stdlib/src/contextvars.rs b/crates/stdlib/src/contextvars.rs index f88ce398c1c..731f5d11e0b 100644 --- a/crates/stdlib/src/contextvars.rs +++ b/crates/stdlib/src/contextvars.rs @@ -1,6 +1,6 @@ use crate::vm::{PyRef, VirtualMachine, builtins::PyModule, class::StaticType}; use _contextvars::PyContext; -use std::cell::RefCell; +use core::cell::RefCell; pub(crate) fn make_module(vm: &VirtualMachine) -> PyRef { let module = _contextvars::make_module(vm); @@ -31,13 +31,13 @@ mod _contextvars { protocol::{PyMappingMethods, PySequenceMethods}, types::{AsMapping, AsSequence, Constructor, Hashable, Representable}, }; - use crossbeam_utils::atomic::AtomicCell; - use indexmap::IndexMap; - use std::sync::LazyLock; - use std::{ + use core::{ cell::{Cell, RefCell, UnsafeCell}, sync::atomic::Ordering, }; + use crossbeam_utils::atomic::AtomicCell; + use indexmap::IndexMap; + use std::sync::LazyLock; // TODO: Real hamt implementation type Hamt = IndexMap, PyObjectRef, ahash::RandomState>; @@ -90,11 +90,11 @@ mod _contextvars { } } - fn borrow_vars(&self) -> impl std::ops::Deref + '_ { + fn borrow_vars(&self) -> impl core::ops::Deref + '_ { self.inner.vars.hamt.borrow() } - fn borrow_vars_mut(&self) -> impl std::ops::DerefMut + '_ { + fn borrow_vars_mut(&self) -> impl core::ops::DerefMut + '_ { self.inner.vars.hamt.borrow_mut() } @@ -293,13 +293,13 @@ mod _contextvars { #[pytraverse(skip)] cached: AtomicCell>, #[pytraverse(skip)] - cached_id: std::sync::atomic::AtomicUsize, // cached_tsid in CPython + cached_id: core::sync::atomic::AtomicUsize, // cached_tsid in CPython #[pytraverse(skip)] hash: UnsafeCell, } - impl std::fmt::Debug for ContextVar { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + impl core::fmt::Debug for ContextVar { + fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result { f.debug_struct("ContextVar").finish() } } @@ -308,7 +308,7 @@ mod _contextvars { impl PartialEq for ContextVar { fn eq(&self, other: &Self) -> bool { - std::ptr::eq(self, other) + core::ptr::eq(self, other) } } impl Eq for ContextVar {} @@ -512,9 +512,9 @@ mod _contextvars { } } - impl std::hash::Hash for ContextVar { + impl core::hash::Hash for ContextVar { #[inline] - fn hash(&self, state: &mut H) { + fn hash(&self, state: &mut H) { unsafe { *self.hash.get() }.hash(state) } } diff --git a/crates/stdlib/src/csv.rs b/crates/stdlib/src/csv.rs index a62594a9f1b..4f6cbd76828 100644 --- a/crates/stdlib/src/csv.rs +++ b/crates/stdlib/src/csv.rs @@ -12,12 +12,13 @@ mod _csv { raise_if_stop, types::{Constructor, IterNext, Iterable, SelfIter}, }; + use alloc::fmt; use csv_core::Terminator; use itertools::{self, Itertools}; use parking_lot::Mutex; use rustpython_vm::match_class; + use std::collections::HashMap; use std::sync::LazyLock; - use std::{collections::HashMap, fmt}; #[pyattr] const QUOTE_MINIMAL: i32 = QuoteStyle::Minimal as i32; @@ -1006,7 +1007,7 @@ mod _csv { return Err(new_csv_error(vm, "filed too long to read".to_string())); } prev_end = end; - let s = std::str::from_utf8(&buffer[range.clone()]) + let s = core::str::from_utf8(&buffer[range.clone()]) // not sure if this is possible - the input was all strings .map_err(|_e| vm.new_unicode_decode_error("csv not utf8"))?; // Rustpython TODO! @@ -1116,7 +1117,7 @@ mod _csv { loop { handle_res!(writer.terminator(&mut buffer[buffer_offset..])); } - let s = std::str::from_utf8(&buffer[..buffer_offset]) + let s = core::str::from_utf8(&buffer[..buffer_offset]) .map_err(|_| vm.new_unicode_decode_error("csv not utf8"))?; self.write.call((s,), vm) diff --git a/crates/stdlib/src/faulthandler.rs b/crates/stdlib/src/faulthandler.rs index f45c9909c6f..eba5643b866 100644 --- a/crates/stdlib/src/faulthandler.rs +++ b/crates/stdlib/src/faulthandler.rs @@ -7,13 +7,13 @@ mod decl { PyObjectRef, PyResult, VirtualMachine, builtins::PyFloat, frame::Frame, function::OptionalArg, py_io::Write, }; + use alloc::sync::Arc; + use core::sync::atomic::{AtomicBool, AtomicI32, Ordering}; + use core::time::Duration; use parking_lot::{Condvar, Mutex}; #[cfg(any(unix, windows))] use rustpython_common::os::{get_errno, set_errno}; - use std::sync::Arc; - use std::sync::atomic::{AtomicBool, AtomicI32, Ordering}; use std::thread; - use std::time::Duration; /// fault_handler_t #[cfg(unix)] @@ -40,7 +40,7 @@ mod decl { enabled: false, name, // SAFETY: sigaction is a C struct that can be zero-initialized - previous: unsafe { std::mem::zeroed() }, + previous: unsafe { core::mem::zeroed() }, } } } @@ -144,7 +144,8 @@ mod decl { static mut FRAME_SNAPSHOTS: [FrameSnapshot; MAX_SNAPSHOT_FRAMES] = [FrameSnapshot::EMPTY; MAX_SNAPSHOT_FRAMES]; #[cfg(any(unix, windows))] - static SNAPSHOT_COUNT: std::sync::atomic::AtomicUsize = std::sync::atomic::AtomicUsize::new(0); + static SNAPSHOT_COUNT: core::sync::atomic::AtomicUsize = + core::sync::atomic::AtomicUsize::new(0); // Signal-safe output functions @@ -240,7 +241,7 @@ mod decl { } let thread_id = current_thread_id(); // Use appropriate width based on platform pointer size - dump_hexadecimal(fd, thread_id, std::mem::size_of::() * 2); + dump_hexadecimal(fd, thread_id, core::mem::size_of::() * 2); puts(fd, " (most recent call first):\n"); } @@ -429,7 +430,7 @@ mod decl { } handler.enabled = false; unsafe { - libc::sigaction(handler.signum, &handler.previous, std::ptr::null_mut()); + libc::sigaction(handler.signum, &handler.previous, core::ptr::null_mut()); } } @@ -549,7 +550,7 @@ mod decl { continue; } - let mut action: libc::sigaction = std::mem::zeroed(); + let mut action: libc::sigaction = core::mem::zeroed(); action.sa_sigaction = faulthandler_fatal_error as libc::sighandler_t; // SA_NODEFER flag action.sa_flags = libc::SA_NODEFER; @@ -1051,8 +1052,8 @@ mod decl { #[cfg(not(target_arch = "wasm32"))] unsafe { suppress_crash_report(); - let ptr: *const i32 = std::ptr::null(); - std::ptr::read_volatile(ptr); + let ptr: *const i32 = core::ptr::null(); + core::ptr::read_volatile(ptr); } } @@ -1132,7 +1133,7 @@ mod decl { panic!("Fatal Python error: in new thread"); }); // Wait a bit for the thread to panic - std::thread::sleep(std::time::Duration::from_secs(1)); + std::thread::sleep(core::time::Duration::from_secs(1)); } } @@ -1203,7 +1204,7 @@ mod decl { suppress_crash_report(); unsafe { - RaiseException(args.code, args.flags, 0, std::ptr::null()); + RaiseException(args.code, args.flags, 0, core::ptr::null()); } } } diff --git a/crates/stdlib/src/fcntl.rs b/crates/stdlib/src/fcntl.rs index dc6a0b8171e..822faeeedaa 100644 --- a/crates/stdlib/src/fcntl.rs +++ b/crates/stdlib/src/fcntl.rs @@ -173,7 +173,7 @@ mod fcntl { }; } - let mut l: libc::flock = unsafe { std::mem::zeroed() }; + let mut l: libc::flock = unsafe { core::mem::zeroed() }; l.l_type = if cmd == libc::LOCK_UN { try_into_l_type!(libc::F_UNLCK) } else if (cmd & libc::LOCK_SH) != 0 { diff --git a/crates/stdlib/src/grp.rs b/crates/stdlib/src/grp.rs index 4664d5fc575..9f7e4195509 100644 --- a/crates/stdlib/src/grp.rs +++ b/crates/stdlib/src/grp.rs @@ -10,8 +10,8 @@ mod grp { exceptions, types::PyStructSequence, }; + use core::ptr::NonNull; use nix::unistd; - use std::ptr::NonNull; #[pystruct_sequence_data] struct GroupData { @@ -30,7 +30,7 @@ mod grp { impl GroupData { fn from_unistd_group(group: unistd::Group, vm: &VirtualMachine) -> Self { - let cstr_lossy = |s: std::ffi::CString| { + let cstr_lossy = |s: alloc::ffi::CString| { s.into_string() .unwrap_or_else(|e| e.into_cstring().to_string_lossy().into_owned()) }; diff --git a/crates/stdlib/src/hashlib.rs b/crates/stdlib/src/hashlib.rs index bfde58f43f7..2da47ceb740 100644 --- a/crates/stdlib/src/hashlib.rs +++ b/crates/stdlib/src/hashlib.rs @@ -97,8 +97,8 @@ pub mod _hashlib { pub ctx: PyRwLock, } - impl std::fmt::Debug for PyHasher { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + impl core::fmt::Debug for PyHasher { + fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result { write!(f, "HASH {}", self.name) } } @@ -170,8 +170,8 @@ pub mod _hashlib { ctx: PyRwLock, } - impl std::fmt::Debug for PyHasherXof { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + impl core::fmt::Debug for PyHasherXof { + fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result { write!(f, "HASHXOF {}", self.name) } } diff --git a/crates/stdlib/src/json.rs b/crates/stdlib/src/json.rs index eb6ed3a5f64..a3fd7972126 100644 --- a/crates/stdlib/src/json.rs +++ b/crates/stdlib/src/json.rs @@ -12,9 +12,9 @@ mod _json { protocol::PyIterReturn, types::{Callable, Constructor}, }; + use core::str::FromStr; use malachite_bigint::BigInt; use rustpython_common::wtf8::Wtf8Buf; - use std::str::FromStr; #[pyattr(name = "make_scanner")] #[pyclass(name = "Scanner", traverse)] @@ -216,7 +216,7 @@ mod _json { let mut buf = Vec::::with_capacity(s.len() + 2); machinery::write_json_string(s, ascii_only, &mut buf) // SAFETY: writing to a vec can't fail - .unwrap_or_else(|_| unsafe { std::hint::unreachable_unchecked() }); + .unwrap_or_else(|_| unsafe { core::hint::unreachable_unchecked() }); // SAFETY: we only output valid utf8 from write_json_string unsafe { String::from_utf8_unchecked(buf) } } diff --git a/crates/stdlib/src/lib.rs b/crates/stdlib/src/lib.rs index 4b463e09c73..6b7796c8bad 100644 --- a/crates/stdlib/src/lib.rs +++ b/crates/stdlib/src/lib.rs @@ -6,6 +6,7 @@ #[macro_use] extern crate rustpython_derive; +extern crate alloc; pub mod array; mod binascii; @@ -103,7 +104,7 @@ use rustpython_common as common; use rustpython_vm as vm; use crate::vm::{builtins, stdlib::StdlibInitFunc}; -use std::borrow::Cow; +use alloc::borrow::Cow; pub fn get_module_inits() -> impl Iterator, StdlibInitFunc)> { macro_rules! modules { diff --git a/crates/stdlib/src/locale.rs b/crates/stdlib/src/locale.rs index 6cca8b9123b..c65f861d208 100644 --- a/crates/stdlib/src/locale.rs +++ b/crates/stdlib/src/locale.rs @@ -41,16 +41,14 @@ use libc::localeconv; #[pymodule] mod _locale { + use alloc::ffi::CString; + use core::{ffi::CStr, ptr}; use rustpython_vm::{ PyObjectRef, PyResult, VirtualMachine, builtins::{PyDictRef, PyIntRef, PyListRef, PyStrRef, PyTypeRef}, convert::ToPyException, function::OptionalArg, }; - use std::{ - ffi::{CStr, CString}, - ptr, - }; #[cfg(all( unix, diff --git a/crates/stdlib/src/lzma.rs b/crates/stdlib/src/lzma.rs index 855a5eae562..b18ac3ee69a 100644 --- a/crates/stdlib/src/lzma.rs +++ b/crates/stdlib/src/lzma.rs @@ -8,6 +8,7 @@ mod _lzma { CompressFlushKind, CompressState, CompressStatusKind, Compressor, DecompressArgs, DecompressError, DecompressState, DecompressStatus, Decompressor, }; + use alloc::fmt; #[pyattr] use lzma_sys::{ LZMA_CHECK_CRC32 as CHECK_CRC32, LZMA_CHECK_CRC64 as CHECK_CRC64, @@ -38,7 +39,6 @@ mod _lzma { use rustpython_vm::function::ArgBytesLike; use rustpython_vm::types::Constructor; use rustpython_vm::{Py, PyObjectRef, PyPayload, PyResult, VirtualMachine}; - use std::fmt; use xz2::stream::{Action, Check, Error, Filters, LzmaOptions, Status, Stream}; #[cfg(windows)] diff --git a/crates/stdlib/src/math.rs b/crates/stdlib/src/math.rs index 62b0ef73ad3..6e139530804 100644 --- a/crates/stdlib/src/math.rs +++ b/crates/stdlib/src/math.rs @@ -10,15 +10,15 @@ mod math { function::{ArgIndex, ArgIntoFloat, ArgIterable, Either, OptionalArg, PosArgs}, identifier, }; + use core::cmp::Ordering; use itertools::Itertools; use malachite_bigint::BigInt; use num_traits::{One, Signed, ToPrimitive, Zero}; use rustpython_common::{float_ops, int::true_div}; - use std::cmp::Ordering; // Constants #[pyattr] - use std::f64::consts::{E as e, PI as pi, TAU as tau}; + use core::f64::consts::{E as e, PI as pi, TAU as tau}; use super::pymath_error_to_exception; #[pyattr(name = "inf")] @@ -136,7 +136,7 @@ mod math { #[pyfunction] fn log(x: PyObjectRef, base: OptionalArg, vm: &VirtualMachine) -> PyResult { - let base = base.map(|b| *b).unwrap_or(std::f64::consts::E); + let base = base.map(|b| *b).unwrap_or(core::f64::consts::E); if base.is_sign_negative() { return Err(vm.new_value_error("math domain error")); } @@ -359,9 +359,9 @@ mod math { .iter() .copied() .map(|x| (x / scale).powi(2)) - .chain(std::iter::once(-norm * norm)) + .chain(core::iter::once(-norm * norm)) // Pairwise summation of floats gives less rounding error than a naive sum. - .tree_reduce(std::ops::Add::add) + .tree_reduce(core::ops::Add::add) .expect("expected at least 1 element"); norm = norm + correction / (2.0 * norm); } @@ -424,12 +424,12 @@ mod math { #[pyfunction] fn degrees(x: ArgIntoFloat) -> f64 { - *x * (180.0 / std::f64::consts::PI) + *x * (180.0 / core::f64::consts::PI) } #[pyfunction] fn radians(x: ArgIntoFloat) -> f64 { - *x * (std::f64::consts::PI / 180.0) + *x * (core::f64::consts::PI / 180.0) } // Hyperbolic functions: @@ -684,7 +684,7 @@ mod math { for j in 0..partials.len() { let mut y: f64 = partials[j]; if x.abs() < y.abs() { - std::mem::swap(&mut x, &mut y); + core::mem::swap(&mut x, &mut y); } // Rounded `x+y` is stored in `hi` with round-off stored in // `lo`. Together `hi+lo` are exactly equal to `x+y`. diff --git a/crates/stdlib/src/mmap.rs b/crates/stdlib/src/mmap.rs index f1e2c2a039d..b520eb2a1a7 100644 --- a/crates/stdlib/src/mmap.rs +++ b/crates/stdlib/src/mmap.rs @@ -21,11 +21,11 @@ mod mmap { sliceable::{SaturatedSlice, SequenceIndex, SequenceIndexOp}, types::{AsBuffer, AsMapping, AsSequence, Constructor, Representable}, }; + use core::ops::{Deref, DerefMut}; use crossbeam_utils::atomic::AtomicCell; use memmap2::{Mmap, MmapMut, MmapOptions}; use num_traits::Signed; use std::io::{self, Write}; - use std::ops::{Deref, DerefMut}; #[cfg(unix)] use nix::{sys::stat::fstat, unistd}; @@ -1057,7 +1057,7 @@ mod mmap { // 3. Replace the old mmap let old_size = self.size.load(); - let copy_size = std::cmp::min(old_size, newsize); + let copy_size = core::cmp::min(old_size, newsize); // Create new anonymous mmap let mut new_mmap_opts = MmapOptions::new(); diff --git a/crates/stdlib/src/opcode.rs b/crates/stdlib/src/opcode.rs index c355b59df91..bd4b9aa750a 100644 --- a/crates/stdlib/src/opcode.rs +++ b/crates/stdlib/src/opcode.rs @@ -8,7 +8,7 @@ mod opcode { bytecode::Instruction, match_class, }; - use std::ops::Deref; + use core::ops::Deref; struct Opcode(Instruction); diff --git a/crates/stdlib/src/openssl.rs b/crates/stdlib/src/openssl.rs index d352d15a614..38103a9ab05 100644 --- a/crates/stdlib/src/openssl.rs +++ b/crates/stdlib/src/openssl.rs @@ -522,10 +522,10 @@ mod _ssl { // Thread-local storage for VirtualMachine pointer during handshake // SNI callback is only called during handshake which is synchronous thread_local! { - static HANDSHAKE_VM: std::cell::Cell> = const { std::cell::Cell::new(None) }; + static HANDSHAKE_VM: core::cell::Cell> = const { core::cell::Cell::new(None) }; // SSL pointer during handshake - needed because connection lock is held during handshake // and callbacks may need to access SSL without acquiring the lock - static HANDSHAKE_SSL_PTR: std::cell::Cell> = const { std::cell::Cell::new(None) }; + static HANDSHAKE_SSL_PTR: core::cell::Cell> = const { core::cell::Cell::new(None) }; } // RAII guard to set/clear thread-local handshake context @@ -1896,7 +1896,7 @@ mod _ssl { ))); return Err(openssl::error::ErrorStack::get()); } - let len = std::cmp::min(pw.len(), buf.len()); + let len = core::cmp::min(pw.len(), buf.len()); buf[..len].copy_from_slice(&pw[..len]); Ok(len) } @@ -2714,7 +2714,7 @@ mod _ssl { // Use thread-local SSL pointer during handshake to avoid deadlock let ssl_ptr = get_ssl_ptr_for_context_change(&self.connection); unsafe { - let mut out: *const libc::c_uchar = std::ptr::null(); + let mut out: *const libc::c_uchar = core::ptr::null(); let mut outlen: libc::c_uint = 0; sys::SSL_get0_alpn_selected(ssl_ptr, &mut out, &mut outlen); diff --git a/crates/stdlib/src/overlapped.rs b/crates/stdlib/src/overlapped.rs index d8f14baf35e..1c74ee271b9 100644 --- a/crates/stdlib/src/overlapped.rs +++ b/crates/stdlib/src/overlapped.rs @@ -35,7 +35,7 @@ mod _overlapped { #[pyattr] const INVALID_HANDLE_VALUE: isize = - unsafe { std::mem::transmute(windows_sys::Win32::Foundation::INVALID_HANDLE_VALUE) }; + unsafe { core::mem::transmute(windows_sys::Win32::Foundation::INVALID_HANDLE_VALUE) }; #[pyattr] const NULL: isize = 0; @@ -57,8 +57,8 @@ mod _overlapped { unsafe impl Sync for OverlappedInner {} unsafe impl Send for OverlappedInner {} - impl std::fmt::Debug for Overlapped { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + impl core::fmt::Debug for Overlapped { + fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result { let zelf = self.inner.lock(); f.debug_struct("Overlapped") // .field("overlapped", &(self.overlapped as *const _ as usize)) @@ -98,8 +98,8 @@ mod _overlapped { address_length: libc::c_int, } - impl std::fmt::Debug for OverlappedReadFrom { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + impl core::fmt::Debug for OverlappedReadFrom { + fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result { f.debug_struct("OverlappedReadFrom") .field("result", &self.result) .field("allocated_buffer", &self.allocated_buffer) @@ -119,8 +119,8 @@ mod _overlapped { address_length: libc::c_int, } - impl std::fmt::Debug for OverlappedReadFromInto { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + impl core::fmt::Debug for OverlappedReadFromInto { + fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result { f.debug_struct("OverlappedReadFromInto") .field("result", &self.result) .field("user_buffer", &self.user_buffer) @@ -226,7 +226,7 @@ mod _overlapped { } #[cfg(target_pointer_width = "32")] - let size = std::cmp::min(size, std::isize::MAX as _); + let size = core::cmp::min(size, std::isize::MAX as _); let buf = vec![0u8; std::cmp::max(size, 1) as usize]; let buf = vm.ctx.new_bytes(buf); @@ -272,10 +272,10 @@ mod _overlapped { if event == INVALID_HANDLE_VALUE { event = unsafe { windows_sys::Win32::System::Threading::CreateEventA( - std::ptr::null(), + core::ptr::null(), Foundation::TRUE, Foundation::FALSE, - std::ptr::null(), + core::ptr::null(), ) as isize }; if event == NULL { @@ -378,11 +378,11 @@ mod _overlapped { let name = widestring::WideCString::from_str(&name).unwrap(); name.as_ptr() } - None => std::ptr::null(), + None => core::ptr::null(), }; let event = unsafe { windows_sys::Win32::System::Threading::CreateEventW( - std::ptr::null(), + core::ptr::null(), manual_reset as _, initial_state as _, name, diff --git a/crates/stdlib/src/posixshmem.rs b/crates/stdlib/src/posixshmem.rs index 2957f16792c..53bf372532d 100644 --- a/crates/stdlib/src/posixshmem.rs +++ b/crates/stdlib/src/posixshmem.rs @@ -4,7 +4,7 @@ pub(crate) use _posixshmem::make_module; #[cfg(all(unix, not(target_os = "redox"), not(target_os = "android")))] #[pymodule] mod _posixshmem { - use std::ffi::CString; + use alloc::ffi::CString; use crate::{ common::os::errno_io_error, diff --git a/crates/stdlib/src/posixsubprocess.rs b/crates/stdlib/src/posixsubprocess.rs index 4b895d2102b..1ebf6619c24 100644 --- a/crates/stdlib/src/posixsubprocess.rs +++ b/crates/stdlib/src/posixsubprocess.rs @@ -13,15 +13,15 @@ use nix::{ unistd::{self, Pid}, }; use std::{ - convert::Infallible as Never, - ffi::{CStr, CString}, io::prelude::*, - marker::PhantomData, - ops::Deref, os::fd::{AsFd, AsRawFd, BorrowedFd, IntoRawFd, OwnedFd, RawFd}, }; use unistd::{Gid, Uid}; +use alloc::ffi::CString; + +use core::{convert::Infallible as Never, ffi::CStr, marker::PhantomData, ops::Deref}; + pub(crate) use _posixsubprocess::make_module; #[pymodule] @@ -87,7 +87,7 @@ impl<'a, T: AsRef> FromIterator<&'a T> for CharPtrVec<'a> { let vec = iter .into_iter() .map(|x| x.as_ref().as_ptr()) - .chain(std::iter::once(std::ptr::null())) + .chain(core::iter::once(core::ptr::null())) .collect(); Self { vec, diff --git a/crates/stdlib/src/resource.rs b/crates/stdlib/src/resource.rs index 052f45e0cad..e6df75e4b01 100644 --- a/crates/stdlib/src/resource.rs +++ b/crates/stdlib/src/resource.rs @@ -9,7 +9,8 @@ mod resource { convert::{ToPyException, ToPyObject}, types::PyStructSequence, }; - use std::{io, mem}; + use core::mem; + use std::io; cfg_if::cfg_if! { if #[cfg(target_os = "android")] { diff --git a/crates/stdlib/src/scproxy.rs b/crates/stdlib/src/scproxy.rs index 1974e7814ae..40267579029 100644 --- a/crates/stdlib/src/scproxy.rs +++ b/crates/stdlib/src/scproxy.rs @@ -22,7 +22,7 @@ mod _scproxy { fn proxy_dict() -> Option> { // Py_BEGIN_ALLOW_THREADS - let proxy_dict = unsafe { SCDynamicStoreCopyProxies(std::ptr::null()) }; + let proxy_dict = unsafe { SCDynamicStoreCopyProxies(core::ptr::null()) }; // Py_END_ALLOW_THREADS if proxy_dict.is_null() { None diff --git a/crates/stdlib/src/select.rs b/crates/stdlib/src/select.rs index 5639a66d2cc..3c2f5e63c7c 100644 --- a/crates/stdlib/src/select.rs +++ b/crates/stdlib/src/select.rs @@ -4,7 +4,8 @@ use crate::vm::{ PyObject, PyObjectRef, PyRef, PyResult, TryFromObject, VirtualMachine, builtins::PyListRef, builtins::PyModule, }; -use std::{io, mem}; +use core::mem; +use std::io; pub(crate) fn make_module(vm: &VirtualMachine) -> PyRef { #[cfg(windows)] @@ -158,7 +159,7 @@ impl FdSet { pub fn new() -> Self { // it's just ints, and all the code that's actually // interacting with it is in C, so it's safe to zero - let mut fdset = std::mem::MaybeUninit::zeroed(); + let mut fdset = core::mem::MaybeUninit::zeroed(); unsafe { platform::FD_ZERO(fdset.as_mut_ptr()) }; Self(fdset) } @@ -191,7 +192,7 @@ pub fn select( ) -> io::Result { let timeout = match timeout { Some(tv) => tv as *mut timeval, - None => std::ptr::null_mut(), + None => core::ptr::null_mut(), }; let ret = unsafe { platform::select( @@ -336,12 +337,10 @@ mod decl { function::OptionalArg, stdlib::io::Fildes, }; + use core::{convert::TryFrom, time::Duration}; use libc::pollfd; use num_traits::{Signed, ToPrimitive}; - use std::{ - convert::TryFrom, - time::{Duration, Instant}, - }; + use std::time::Instant; #[derive(Default)] pub(super) struct TimeoutArg(pub Option); @@ -554,8 +553,8 @@ mod decl { stdlib::io::Fildes, types::Constructor, }; + use core::ops::Deref; use rustix::event::epoll::{self, EventData, EventFlags}; - use std::ops::Deref; use std::os::fd::{AsRawFd, IntoRawFd, OwnedFd}; use std::time::Instant; diff --git a/crates/stdlib/src/socket.rs b/crates/stdlib/src/socket.rs index 91c7b1201f1..71bc8e9f170 100644 --- a/crates/stdlib/src/socket.rs +++ b/crates/stdlib/src/socket.rs @@ -22,15 +22,19 @@ mod _socket { types::{Constructor, DefaultConstructor, Initializer, Representable}, utils::ToCString, }; + use core::{ + mem::MaybeUninit, + net::{Ipv4Addr, Ipv6Addr, SocketAddr}, + time::Duration, + }; use crossbeam_utils::atomic::AtomicCell; use num_traits::ToPrimitive; use socket2::Socket; use std::{ ffi, io::{self, Read, Write}, - mem::MaybeUninit, - net::{self, Ipv4Addr, Ipv6Addr, Shutdown, SocketAddr, ToSocketAddrs}, - time::{Duration, Instant}, + net::{self, Shutdown, ToSocketAddrs}, + time::Instant, }; #[cfg(unix)] @@ -934,7 +938,7 @@ mod _socket { sock: PyRwLock>, } - const _: () = assert!(std::mem::size_of::>() == std::mem::size_of::()); + const _: () = assert!(core::mem::size_of::>() == core::mem::size_of::()); impl Default for PySocket { fn default() -> Self { @@ -1494,7 +1498,7 @@ mod _socket { Some(errcode!(ENOTSOCK)) | Some(errcode!(EBADF)) ) => { - std::mem::forget(sock); + core::mem::forget(sock); return Err(e.into()); } _ => {} @@ -2052,7 +2056,7 @@ mod _socket { cmsgs: &[(i32, i32, ArgBytesLike)], vm: &VirtualMachine, ) -> PyResult> { - use std::{mem, ptr}; + use core::{mem, ptr}; if cmsgs.is_empty() { return Ok(vec![]); @@ -2210,7 +2214,7 @@ mod _socket { let buflen = buflen.unwrap_or(0); if buflen == 0 { let mut flag: libc::c_int = 0; - let mut flagsize = std::mem::size_of::() as _; + let mut flagsize = core::mem::size_of::() as _; let ret = unsafe { c::getsockopt( fd as _, @@ -2270,11 +2274,11 @@ mod _socket { level, name, val as *const i32 as *const _, - std::mem::size_of::() as _, + core::mem::size_of::() as _, ) }, (None, OptionalArg::Present(optlen)) => unsafe { - c::setsockopt(fd as _, level, name, std::ptr::null(), optlen as _) + c::setsockopt(fd as _, level, name, core::ptr::null(), optlen as _) }, _ => { return Err(vm @@ -2456,7 +2460,7 @@ mod _socket { } impl ToSocketAddrs for Address { - type Iter = std::vec::IntoIter; + type Iter = alloc::vec::IntoIter; fn to_socket_addrs(&self) -> io::Result { (self.host.as_str(), self.port).to_socket_addrs() } @@ -2616,7 +2620,7 @@ mod _socket { } fn cstr_opt_as_ptr(x: &OptionalArg) -> *const libc::c_char { - x.as_ref().map_or_else(std::ptr::null, |s| s.as_ptr()) + x.as_ref().map_or_else(core::ptr::null, |s| s.as_ptr()) } #[pyfunction] @@ -2807,7 +2811,7 @@ mod _socket { vm.state .codec_registry .encode_text(s.to_owned(), "idna", None, vm)?; - let host_str = std::str::from_utf8(encoded.as_bytes()) + let host_str = core::str::from_utf8(encoded.as_bytes()) .map_err(|_| vm.new_runtime_error("idna output is not utf8".to_owned()))?; Some(host_str.to_owned()) } @@ -3183,7 +3187,7 @@ mod _socket { .state .codec_registry .encode_text(pyname, "idna", None, vm)?; - let name = std::str::from_utf8(name.as_bytes()) + let name = core::str::from_utf8(name.as_bytes()) .map_err(|_| vm.new_runtime_error("idna output is not utf8"))?; let mut res = dns_lookup::getaddrinfo(Some(name), None, Some(hints)) .map_err(|e| convert_socket_error(vm, e, SocketError::GaiError))?; @@ -3339,7 +3343,7 @@ mod _socket { #[pyfunction] fn dup(x: PyObjectRef, vm: &VirtualMachine) -> Result { let sock = get_raw_sock(x, vm)?; - let sock = std::mem::ManuallyDrop::new(sock_from_raw(sock, vm)?); + let sock = core::mem::ManuallyDrop::new(sock_from_raw(sock, vm)?); let newsock = sock.try_clone()?; let fd = into_sock_fileno(newsock); #[cfg(windows)] diff --git a/crates/stdlib/src/sqlite.rs b/crates/stdlib/src/sqlite.rs index 103a827e99a..3a82787cd8f 100644 --- a/crates/stdlib/src/sqlite.rs +++ b/crates/stdlib/src/sqlite.rs @@ -844,7 +844,7 @@ mod _sqlite { } impl Debug for Connection { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> Result<(), std::fmt::Error> { + fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> Result<(), std::fmt::Error> { write!(f, "Sqlite3 Connection") } } @@ -2583,7 +2583,7 @@ mod _sqlite { } impl Debug for Statement { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> Result<(), std::fmt::Error> { + fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> Result<(), std::fmt::Error> { write!( f, "{} Statement", diff --git a/crates/stdlib/src/ssl.rs b/crates/stdlib/src/ssl.rs index 16449e2d019..b90176a62fa 100644 --- a/crates/stdlib/src/ssl.rs +++ b/crates/stdlib/src/ssl.rs @@ -52,14 +52,12 @@ mod _ssl { use super::error::{ PySSLEOFError, PySSLError, create_ssl_want_read_error, create_ssl_want_write_error, }; - use std::{ - collections::HashMap, - sync::{ - Arc, - atomic::{AtomicUsize, Ordering}, - }, - time::{Duration, SystemTime}, + use alloc::sync::Arc; + use core::{ + sync::atomic::{AtomicUsize, Ordering}, + time::Duration, }; + use std::{collections::HashMap, time::SystemTime}; // Rustls imports use parking_lot::{Mutex as ParkingMutex, RwLock as ParkingRwLock}; @@ -3124,7 +3122,7 @@ mod _ssl { // When server_hostname=None, use an IP address to suppress SNI // no hostname = no SNI extension ServerName::IpAddress( - std::net::IpAddr::V4(std::net::Ipv4Addr::new(127, 0, 0, 1)).into(), + core::net::IpAddr::V4(core::net::Ipv4Addr::new(127, 0, 0, 1)).into(), ) }; @@ -3385,7 +3383,7 @@ mod _ssl { let mut written = 0; while written < data.len() { - let chunk_end = std::cmp::min(written + CHUNK_SIZE, data.len()); + let chunk_end = core::cmp::min(written + CHUNK_SIZE, data.len()); let chunk = &data[written..chunk_end]; // Write chunk to TLS layer @@ -4176,8 +4174,8 @@ mod _ssl { #[pygetset] fn id(&self, vm: &VirtualMachine) -> PyBytesRef { // Return session ID (hash of session data for uniqueness) + use core::hash::{Hash, Hasher}; use std::collections::hash_map::DefaultHasher; - use std::hash::{Hash, Hasher}; let mut hasher = DefaultHasher::new(); self.session_data.hash(&mut hasher); @@ -4487,7 +4485,7 @@ mod _ssl { let mut result = Vec::new(); - let mut crl_context: *const CRL_CONTEXT = std::ptr::null(); + let mut crl_context: *const CRL_CONTEXT = core::ptr::null(); loop { crl_context = unsafe { CertEnumCRLsInStore(store, crl_context) }; if crl_context.is_null() { @@ -4587,8 +4585,8 @@ mod _ssl { // Implement Hashable trait for PySSLCertificate impl Hashable for PySSLCertificate { fn hash(zelf: &Py, _vm: &VirtualMachine) -> PyResult { + use core::hash::{Hash, Hasher}; use std::collections::hash_map::DefaultHasher; - use std::hash::{Hash, Hasher}; let mut hasher = DefaultHasher::new(); zelf.der_bytes.hash(&mut hasher); diff --git a/crates/stdlib/src/ssl/cert.rs b/crates/stdlib/src/ssl/cert.rs index b3cb7d6c14e..cd39972cf41 100644 --- a/crates/stdlib/src/ssl/cert.rs +++ b/crates/stdlib/src/ssl/cert.rs @@ -9,6 +9,7 @@ //! - Building and verifying certificate chains //! - Loading certificates from files, directories, and bytes +use alloc::sync::Arc; use chrono::{DateTime, Utc}; use parking_lot::RwLock as ParkingRwLock; use rustls::{ @@ -19,7 +20,6 @@ use rustls::{ }; use rustpython_vm::{PyObjectRef, PyResult, VirtualMachine}; use std::collections::HashSet; -use std::sync::Arc; use x509_parser::prelude::*; use super::compat::{VERIFY_X509_PARTIAL_CHAIN, VERIFY_X509_STRICT}; @@ -51,8 +51,9 @@ const ALL_SIGNATURE_SCHEMES: &[SignatureScheme] = &[ /// operations, reducing code duplication and ensuring uniform error messages /// across the codebase. mod cert_error { + use alloc::sync::Arc; + use core::fmt::{Debug, Display}; use std::io; - use std::sync::Arc; /// Create InvalidData error with formatted message pub fn invalid_data(msg: impl Into) -> io::Error { @@ -67,11 +68,11 @@ mod cert_error { invalid_data(format!("no start line: {context}")) } - pub fn parse_failed(e: impl std::fmt::Display) -> io::Error { + pub fn parse_failed(e: impl Display) -> io::Error { invalid_data(format!("Failed to parse PEM certificate: {e}")) } - pub fn parse_failed_debug(e: impl std::fmt::Debug) -> io::Error { + pub fn parse_failed_debug(e: impl Debug) -> io::Error { invalid_data(format!("Failed to parse PEM certificate: {e:?}")) } @@ -88,7 +89,7 @@ mod cert_error { invalid_data(format!("not enough data: {context}")) } - pub fn parse_failed(e: impl std::fmt::Display) -> io::Error { + pub fn parse_failed(e: impl Display) -> io::Error { invalid_data(format!("Failed to parse DER certificate: {e}")) } } @@ -101,15 +102,15 @@ mod cert_error { invalid_data(format!("No private key found in {context}")) } - pub fn parse_failed(e: impl std::fmt::Display) -> io::Error { + pub fn parse_failed(e: impl Display) -> io::Error { invalid_data(format!("Failed to parse private key: {e}")) } - pub fn parse_encrypted_failed(e: impl std::fmt::Display) -> io::Error { + pub fn parse_encrypted_failed(e: impl Display) -> io::Error { invalid_data(format!("Failed to parse encrypted private key: {e}")) } - pub fn decrypt_failed(e: impl std::fmt::Display) -> io::Error { + pub fn decrypt_failed(e: impl Display) -> io::Error { io::Error::other(format!( "Failed to decrypt private key (wrong password?): {e}", )) @@ -383,7 +384,7 @@ pub fn cert_der_to_dict_helper(vm: &VirtualMachine, cert_der: &[u8]) -> PyResult s.to_string() } else { let value_bytes = attr.attr_value().data; - match std::str::from_utf8(value_bytes) { + match core::str::from_utf8(value_bytes) { Ok(s) => s.to_string(), Err(_) => String::from_utf8_lossy(value_bytes).into_owned(), } @@ -1126,7 +1127,7 @@ pub(super) fn load_cert_chain_from_file( cert_path: &str, key_path: &str, password: Option<&str>, -) -> Result<(Vec>, PrivateKeyDer<'static>), Box> { +) -> Result<(Vec>, PrivateKeyDer<'static>), Box> { // Load certificate file - preserve io::Error for errno let cert_contents = std::fs::read(cert_path)?; @@ -1727,13 +1728,13 @@ fn verify_ip_address( cert: &X509Certificate<'_>, expected_ip: &rustls::pki_types::IpAddr, ) -> Result<(), rustls::Error> { - use std::net::IpAddr; + use core::net::IpAddr; use x509_parser::extensions::GeneralName; // Convert rustls IpAddr to std::net::IpAddr for comparison let expected_std_ip: IpAddr = match expected_ip { - rustls::pki_types::IpAddr::V4(octets) => IpAddr::V4(std::net::Ipv4Addr::from(*octets)), - rustls::pki_types::IpAddr::V6(octets) => IpAddr::V6(std::net::Ipv6Addr::from(*octets)), + rustls::pki_types::IpAddr::V4(octets) => IpAddr::V4(core::net::Ipv4Addr::from(*octets)), + rustls::pki_types::IpAddr::V6(octets) => IpAddr::V6(core::net::Ipv6Addr::from(*octets)), }; // Check Subject Alternative Names for IP addresses @@ -1745,7 +1746,7 @@ fn verify_ip_address( 4 => { // IPv4 if let Ok(octets) = <[u8; 4]>::try_from(*cert_ip_bytes) { - IpAddr::V4(std::net::Ipv4Addr::from(octets)) + IpAddr::V4(core::net::Ipv4Addr::from(octets)) } else { continue; } @@ -1753,7 +1754,7 @@ fn verify_ip_address( 16 => { // IPv6 if let Ok(octets) = <[u8; 16]>::try_from(*cert_ip_bytes) { - IpAddr::V6(std::net::Ipv6Addr::from(octets)) + IpAddr::V6(core::net::Ipv6Addr::from(octets)) } else { continue; } diff --git a/crates/stdlib/src/ssl/compat.rs b/crates/stdlib/src/ssl/compat.rs index fa12855e242..2168fcfc91f 100644 --- a/crates/stdlib/src/ssl/compat.rs +++ b/crates/stdlib/src/ssl/compat.rs @@ -13,6 +13,7 @@ mod ssl_data; use crate::socket::{SelectKind, timeout_error_msg}; use crate::vm::VirtualMachine; +use alloc::sync::Arc; use parking_lot::RwLock as ParkingRwLock; use rustls::RootCertStore; use rustls::client::ClientConfig; @@ -28,7 +29,7 @@ use rustpython_vm::convert::IntoPyException; use rustpython_vm::function::ArgBytesLike; use rustpython_vm::{AsObject, Py, PyObjectRef, PyPayload, PyResult, TryFromObject}; use std::io::Read; -use std::sync::{Arc, Once}; +use std::sync::Once; // Import PySSLSocket from parent module use super::_ssl::PySSLSocket; diff --git a/crates/stdlib/src/syslog.rs b/crates/stdlib/src/syslog.rs index d0ed3f60949..b52d1415692 100644 --- a/crates/stdlib/src/syslog.rs +++ b/crates/stdlib/src/syslog.rs @@ -11,7 +11,8 @@ mod syslog { function::{OptionalArg, OptionalOption}, utils::ToCString, }; - use std::{ffi::CStr, os::raw::c_char}; + use core::ffi::CStr; + use std::os::raw::c_char; #[pyattr] use libc::{ @@ -50,7 +51,7 @@ mod syslog { fn as_ptr(&self) -> *const c_char { match self { Self::Explicit(cstr) => cstr.as_ptr(), - Self::Implicit => std::ptr::null(), + Self::Implicit => core::ptr::null(), } } } diff --git a/crates/stdlib/src/tkinter.rs b/crates/stdlib/src/tkinter.rs index 687458b193b..49dcdc5f84f 100644 --- a/crates/stdlib/src/tkinter.rs +++ b/crates/stdlib/src/tkinter.rs @@ -59,8 +59,8 @@ mod _tkinter { value: *mut tk_sys::Tcl_Obj, } - impl std::fmt::Debug for TclObject { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + impl core::fmt::Debug for TclObject { + fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result { write!(f, "TclObject") } } @@ -107,8 +107,8 @@ mod _tkinter { unsafe impl Send for TkApp {} unsafe impl Sync for TkApp {} - impl std::fmt::Debug for TkApp { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + impl core::fmt::Debug for TkApp { + fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result { write!(f, "TkApp") } } diff --git a/crates/stdlib/src/zlib.rs b/crates/stdlib/src/zlib.rs index 9ca94939f78..632543c5c64 100644 --- a/crates/stdlib/src/zlib.rs +++ b/crates/stdlib/src/zlib.rs @@ -39,7 +39,7 @@ mod zlib { #[pyattr(name = "ZLIB_RUNTIME_VERSION")] #[pyattr] const ZLIB_VERSION: &str = unsafe { - match std::ffi::CStr::from_ptr(libz_sys::zlibVersion()).to_str() { + match core::ffi::CStr::from_ptr(libz_sys::zlibVersion()).to_str() { Ok(s) => s, Err(_) => unreachable!(), } @@ -322,7 +322,7 @@ mod zlib { }; let inner = &mut *self.inner.lock(); - let data = std::mem::replace(&mut inner.unconsumed_tail, vm.ctx.empty_bytes.clone()); + let data = core::mem::replace(&mut inner.unconsumed_tail, vm.ctx.empty_bytes.clone()); let (ret, _) = Self::decompress_inner(inner, &data, length, None, true, vm)?; diff --git a/crates/venvlauncher/src/main.rs b/crates/venvlauncher/src/main.rs index aaf584dfa87..fe147ce7ff3 100644 --- a/crates/venvlauncher/src/main.rs +++ b/crates/venvlauncher/src/main.rs @@ -22,7 +22,7 @@ fn main() -> ExitCode { } } -fn run() -> Result> { +fn run() -> Result> { // 1. Get own executable path let exe_path = env::current_exe()?; let exe_name = exe_path @@ -72,7 +72,7 @@ fn run() -> Result> { } /// Parse the `home=` value from pyvenv.cfg -fn read_home(cfg_path: &Path) -> Result> { +fn read_home(cfg_path: &Path) -> Result> { let content = fs::read_to_string(cfg_path)?; for line in content.lines() { @@ -95,7 +95,7 @@ fn read_home(cfg_path: &Path) -> Result> { } /// Launch the Python process and wait for it to complete -fn launch_process(exe: &Path, args: &[String]) -> Result> { +fn launch_process(exe: &Path, args: &[String]) -> Result> { use std::process::Command; let status = Command::new(exe).args(args).status()?; diff --git a/crates/vm/src/anystr.rs b/crates/vm/src/anystr.rs index ef6d24c100e..79b62a58abf 100644 --- a/crates/vm/src/anystr.rs +++ b/crates/vm/src/anystr.rs @@ -6,6 +6,8 @@ use crate::{ }; use num_traits::{cast::ToPrimitive, sign::Signed}; +use core::ops::Range; + #[derive(FromArgs)] pub struct SplitArgs { #[pyarg(any, default)] @@ -43,7 +45,7 @@ pub struct StartsEndsWithArgs { } impl StartsEndsWithArgs { - pub fn get_value(self, len: usize) -> (PyObjectRef, Option>) { + pub fn get_value(self, len: usize) -> (PyObjectRef, Option>) { let range = if self.start.is_some() || self.end.is_some() { Some(adjust_indices(self.start, self.end, len)) } else { @@ -56,7 +58,7 @@ impl StartsEndsWithArgs { pub fn prepare(self, s: &S, len: usize, substr: F) -> Option<(PyObjectRef, &S)> where S: ?Sized + AnyStr, - F: Fn(&S, std::ops::Range) -> &S, + F: Fn(&S, Range) -> &S, { let (affix, range) = self.get_value(len); let substr = if let Some(range) = range { @@ -83,11 +85,7 @@ fn saturate_to_isize(py_int: PyIntRef) -> isize { } // help get optional string indices -pub fn adjust_indices( - start: Option, - end: Option, - len: usize, -) -> std::ops::Range { +pub fn adjust_indices(start: Option, end: Option, len: usize) -> Range { let mut start = start.map_or(0, saturate_to_isize); let mut end = end.map_or(len as isize, saturate_to_isize); if end > len as isize { @@ -111,7 +109,7 @@ pub trait StringRange { fn is_normal(&self) -> bool; } -impl StringRange for std::ops::Range { +impl StringRange for Range { fn is_normal(&self) -> bool { self.start <= self.end } @@ -144,9 +142,9 @@ pub trait AnyStr { fn to_container(&self) -> Self::Container; fn as_bytes(&self) -> &[u8]; fn elements(&self) -> impl Iterator; - fn get_bytes(&self, range: std::ops::Range) -> &Self; + fn get_bytes(&self, range: Range) -> &Self; // FIXME: get_chars is expensive for str - fn get_chars(&self, range: std::ops::Range) -> &Self; + fn get_chars(&self, range: Range) -> &Self; fn bytes_len(&self) -> usize; // NOTE: str::chars().count() consumes the O(n) time. But pystr::char_len does cache. // So using chars_len directly is too expensive and the below method shouldn't be implemented. @@ -254,7 +252,7 @@ pub trait AnyStr { } #[inline] - fn py_find(&self, needle: &Self, range: std::ops::Range, find: F) -> Option + fn py_find(&self, needle: &Self, range: Range, find: F) -> Option where F: Fn(&Self, &Self) -> Option, { @@ -268,7 +266,7 @@ pub trait AnyStr { } #[inline] - fn py_count(&self, needle: &Self, range: std::ops::Range, count: F) -> usize + fn py_count(&self, needle: &Self, range: Range, count: F) -> usize where F: Fn(&Self, &Self) -> usize, { @@ -283,9 +281,9 @@ pub trait AnyStr { let mut u = Self::Container::with_capacity( (left + right) * fillchar.bytes_len() + self.bytes_len(), ); - u.extend(std::iter::repeat_n(fillchar, left)); + u.extend(core::iter::repeat_n(fillchar, left)); u.push_str(self); - u.extend(std::iter::repeat_n(fillchar, right)); + u.extend(core::iter::repeat_n(fillchar, right)); u } @@ -305,7 +303,7 @@ pub trait AnyStr { fn py_join( &self, - mut iter: impl std::iter::Iterator + TryFromObject>>, + mut iter: impl core::iter::Iterator + TryFromObject>>, ) -> PyResult { let mut joined = if let Some(elem) = iter.next() { elem?.as_ref().unwrap().to_container() @@ -328,7 +326,7 @@ pub trait AnyStr { ) -> PyResult<(Self::Container, bool, Self::Container)> where F: Fn() -> S, - S: std::iter::Iterator, + S: core::iter::Iterator, { if sub.is_empty() { return Err(vm.new_value_error("empty separator")); diff --git a/crates/vm/src/buffer.rs b/crates/vm/src/buffer.rs index 5c67f87521d..3d5e48015ea 100644 --- a/crates/vm/src/buffer.rs +++ b/crates/vm/src/buffer.rs @@ -5,11 +5,13 @@ use crate::{ convert::ToPyObject, function::{ArgBytesLike, ArgIntoBool, ArgIntoFloat}, }; +use alloc::fmt; +use core::{iter::Peekable, mem}; use half::f16; use itertools::Itertools; use malachite_bigint::BigInt; use num_traits::{PrimInt, ToPrimitive}; -use std::{fmt, iter::Peekable, mem, os::raw}; +use std::os::raw; type PackFunc = fn(&VirtualMachine, PyObjectRef, &mut [u8]) -> PyResult<()>; type UnpackFunc = fn(&VirtualMachine, &[u8]) -> PyObjectRef; @@ -545,7 +547,7 @@ macro_rules! make_pack_prim_int { } #[inline] fn unpack_int(data: &[u8]) -> Self { - let mut x = [0; std::mem::size_of::<$T>()]; + let mut x = [0; core::mem::size_of::<$T>()]; x.copy_from_slice(data); E::convert(<$T>::from_ne_bytes(x)) } @@ -681,7 +683,7 @@ fn pack_pascal(vm: &VirtualMachine, arg: PyObjectRef, buf: &mut [u8]) -> PyResul } let b = ArgBytesLike::try_from_object(vm, arg)?; b.with_ref(|data| { - let string_length = std::cmp::min(std::cmp::min(data.len(), 255), buf.len() - 1); + let string_length = core::cmp::min(core::cmp::min(data.len(), 255), buf.len() - 1); buf[0] = string_length as u8; write_string(&mut buf[1..], data); }); @@ -689,7 +691,7 @@ fn pack_pascal(vm: &VirtualMachine, arg: PyObjectRef, buf: &mut [u8]) -> PyResul } fn write_string(buf: &mut [u8], data: &[u8]) { - let len_from_data = std::cmp::min(data.len(), buf.len()); + let len_from_data = core::cmp::min(data.len(), buf.len()); buf[..len_from_data].copy_from_slice(&data[..len_from_data]); for byte in &mut buf[len_from_data..] { *byte = 0 @@ -708,7 +710,7 @@ fn unpack_pascal(vm: &VirtualMachine, data: &[u8]) -> PyObjectRef { return vm.ctx.new_bytes(vec![]).into(); } }; - let len = std::cmp::min(len as usize, data.len()); + let len = core::cmp::min(len as usize, data.len()); vm.ctx.new_bytes(data[..len].to_vec()).into() } diff --git a/crates/vm/src/builtins/bool.rs b/crates/vm/src/builtins/bool.rs index cfd1f136d14..3dabdbae717 100644 --- a/crates/vm/src/builtins/bool.rs +++ b/crates/vm/src/builtins/bool.rs @@ -8,9 +8,9 @@ use crate::{ protocol::PyNumberMethods, types::{AsNumber, Constructor, Representable}, }; +use core::fmt::{Debug, Formatter}; use malachite_bigint::Sign; use num_traits::Zero; -use std::fmt::{Debug, Formatter}; impl ToPyObject for bool { fn to_pyobject(self, vm: &VirtualMachine) -> PyObjectRef { @@ -90,7 +90,7 @@ impl PyObjectRef { pub struct PyBool(pub PyInt); impl Debug for PyBool { - fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { + fn fmt(&self, f: &mut Formatter<'_>) -> core::fmt::Result { let value = !self.0.as_bigint().is_zero(); write!(f, "PyBool({})", value) } diff --git a/crates/vm/src/builtins/builtin_func.rs b/crates/vm/src/builtins/builtin_func.rs index 2b569375b28..422f922df94 100644 --- a/crates/vm/src/builtins/builtin_func.rs +++ b/crates/vm/src/builtins/builtin_func.rs @@ -7,7 +7,7 @@ use crate::{ function::{FuncArgs, PyComparisonValue, PyMethodDef, PyMethodFlags, PyNativeFn}, types::{Callable, Comparable, PyComparisonOp, Representable}, }; -use std::fmt; +use alloc::fmt; // PyCFunctionObject in CPython #[pyclass(name = "builtin_function_or_method", module = false)] @@ -212,7 +212,7 @@ impl Comparable for PyNativeMethod { (None, None) => true, _ => false, }; - let eq = eq && std::ptr::eq(zelf.func.value, other.func.value); + let eq = eq && core::ptr::eq(zelf.func.value, other.func.value); Ok(eq.into()) } else { Ok(PyComparisonValue::NotImplemented) diff --git a/crates/vm/src/builtins/bytearray.rs b/crates/vm/src/builtins/bytearray.rs index c5861befb73..212e4604ec9 100644 --- a/crates/vm/src/builtins/bytearray.rs +++ b/crates/vm/src/builtins/bytearray.rs @@ -37,7 +37,7 @@ use crate::{ }, }; use bstr::ByteSlice; -use std::mem::size_of; +use core::mem::size_of; #[pyclass(module = false, name = "bytearray", unhashable = true)] #[derive(Debug, Default)] @@ -687,7 +687,7 @@ impl Initializer for PyByteArray { fn init(zelf: PyRef, options: Self::Args, vm: &VirtualMachine) -> PyResult<()> { // First unpack bytearray and *then* get a lock to set it. let mut inner = options.get_bytearray_inner(vm)?; - std::mem::swap(&mut *zelf.inner_mut(), &mut inner); + core::mem::swap(&mut *zelf.inner_mut(), &mut inner); Ok(()) } } diff --git a/crates/vm/src/builtins/bytes.rs b/crates/vm/src/builtins/bytes.rs index 0c67cd7bf24..b3feac8ac97 100644 --- a/crates/vm/src/builtins/bytes.rs +++ b/crates/vm/src/builtins/bytes.rs @@ -29,8 +29,8 @@ use crate::{ }, }; use bstr::ByteSlice; +use core::{mem::size_of, ops::Deref}; use std::sync::LazyLock; -use std::{mem::size_of, ops::Deref}; #[pyclass(module = false, name = "bytes")] #[derive(Clone, Debug)] diff --git a/crates/vm/src/builtins/code.rs b/crates/vm/src/builtins/code.rs index e46cc711bb3..b897ef9d311 100644 --- a/crates/vm/src/builtins/code.rs +++ b/crates/vm/src/builtins/code.rs @@ -11,10 +11,11 @@ use crate::{ function::OptionalArg, types::{Constructor, Representable}, }; +use alloc::fmt; +use core::{borrow::Borrow, ops::Deref}; use malachite_bigint::BigInt; use num_traits::Zero; use rustpython_compiler_core::{OneIndexed, bytecode::CodeUnits, bytecode::PyCodeLocationInfoKind}; -use std::{borrow::Borrow, fmt, ops::Deref}; /// State for iterating through code address ranges struct PyCodeAddressRange<'a> { @@ -601,7 +602,7 @@ impl PyCode { pub fn co_code(&self, vm: &VirtualMachine) -> crate::builtins::PyBytesRef { // SAFETY: CodeUnit is #[repr(C)] with size 2, so we can safely transmute to bytes let bytes = unsafe { - std::slice::from_raw_parts( + core::slice::from_raw_parts( self.code.instructions.as_ptr() as *const u8, self.code.instructions.len() * 2, ) diff --git a/crates/vm/src/builtins/complex.rs b/crates/vm/src/builtins/complex.rs index ba74d5e0367..78729b2f5c0 100644 --- a/crates/vm/src/builtins/complex.rs +++ b/crates/vm/src/builtins/complex.rs @@ -10,10 +10,10 @@ use crate::{ stdlib::warnings, types::{AsNumber, Comparable, Constructor, Hashable, PyComparisonOp, Representable}, }; +use core::num::Wrapping; use num_complex::Complex64; use num_traits::Zero; use rustpython_common::hash; -use std::num::Wrapping; /// Create a complex number from a real part and an optional imaginary part. /// diff --git a/crates/vm/src/builtins/descriptor.rs b/crates/vm/src/builtins/descriptor.rs index 802b81f6d79..aa9da6e2d44 100644 --- a/crates/vm/src/builtins/descriptor.rs +++ b/crates/vm/src/builtins/descriptor.rs @@ -59,8 +59,8 @@ impl PyPayload for PyMethodDescriptor { } } -impl std::fmt::Debug for PyMethodDescriptor { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { +impl core::fmt::Debug for PyMethodDescriptor { + fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result { write!(f, "method descriptor for '{}'", self.common.name) } } @@ -218,8 +218,8 @@ impl PyMemberDef { } } -impl std::fmt::Debug for PyMemberDef { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { +impl core::fmt::Debug for PyMemberDef { + fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result { f.debug_struct("PyMemberDef") .field("name", &self.name) .field("kind", &self.kind) @@ -445,8 +445,8 @@ pub enum SlotFunc { NumTernaryRight(PyNumberTernaryFunc), // __rpow__ (swapped first two args) } -impl std::fmt::Debug for SlotFunc { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { +impl core::fmt::Debug for SlotFunc { + fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result { match self { SlotFunc::Init(_) => write!(f, "SlotFunc::Init(...)"), SlotFunc::Hash(_) => write!(f, "SlotFunc::Hash(...)"), diff --git a/crates/vm/src/builtins/dict.rs b/crates/vm/src/builtins/dict.rs index 567e18d6419..358685fcdc4 100644 --- a/crates/vm/src/builtins/dict.rs +++ b/crates/vm/src/builtins/dict.rs @@ -23,8 +23,8 @@ use crate::{ }, vm::VirtualMachine, }; +use alloc::fmt; use rustpython_common::lock::PyMutex; -use std::fmt; use std::sync::LazyLock; pub type DictContentType = dict_inner::Dict; @@ -219,7 +219,7 @@ impl PyDict { #[pymethod] fn __sizeof__(&self) -> usize { - std::mem::size_of::() + self.entries.sizeof() + core::mem::size_of::() + self.entries.sizeof() } #[pymethod] @@ -759,7 +759,7 @@ impl ExactSizeIterator for DictIter<'_> { #[pyclass] trait DictView: PyPayload + PyClassDef + Iterable + Representable { - type ReverseIter: PyPayload + std::fmt::Debug; + type ReverseIter: PyPayload + core::fmt::Debug; fn dict(&self) -> &Py; fn item(vm: &VirtualMachine, key: PyObjectRef, value: PyObjectRef) -> PyObjectRef; diff --git a/crates/vm/src/builtins/function.rs b/crates/vm/src/builtins/function.rs index c29e45ddcf6..95d70afcbc7 100644 --- a/crates/vm/src/builtins/function.rs +++ b/crates/vm/src/builtins/function.rs @@ -149,7 +149,7 @@ impl PyFunction { None }; - let arg_pos = |range: std::ops::Range<_>, name: &str| { + let arg_pos = |range: core::ops::Range<_>, name: &str| { code.varnames .iter() .enumerate() @@ -255,7 +255,7 @@ impl PyFunction { } if let Some(defaults) = defaults { - let n = std::cmp::min(nargs, n_expected_args); + let n = core::cmp::min(nargs, n_expected_args); let i = n.saturating_sub(n_required); // We have sufficient defaults, so iterate over the corresponding names and use diff --git a/crates/vm/src/builtins/genericalias.rs b/crates/vm/src/builtins/genericalias.rs index 8a7288980fa..5596aca9da2 100644 --- a/crates/vm/src/builtins/genericalias.rs +++ b/crates/vm/src/builtins/genericalias.rs @@ -16,7 +16,7 @@ use crate::{ PyComparisonOp, Representable, }, }; -use std::fmt; +use alloc::fmt; // attr_exceptions static ATTR_EXCEPTIONS: [&str; 12] = [ diff --git a/crates/vm/src/builtins/getset.rs b/crates/vm/src/builtins/getset.rs index 3fa4667a997..a3f0605a473 100644 --- a/crates/vm/src/builtins/getset.rs +++ b/crates/vm/src/builtins/getset.rs @@ -19,8 +19,8 @@ pub struct PyGetSet { // doc: Option, } -impl std::fmt::Debug for PyGetSet { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { +impl core::fmt::Debug for PyGetSet { + fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result { write!( f, "PyGetSet {{ name: {}, getter: {}, setter: {} }}", @@ -158,7 +158,7 @@ impl Representable for PyGetSet { fn repr_str(zelf: &Py, vm: &VirtualMachine) -> PyResult { let class = unsafe { zelf.class.borrow_static() }; // Special case for object type - if std::ptr::eq(class, vm.ctx.types.object_type) { + if core::ptr::eq(class, vm.ctx.types.object_type) { Ok(format!("", zelf.name)) } else { Ok(format!( diff --git a/crates/vm/src/builtins/int.rs b/crates/vm/src/builtins/int.rs index 37b41e085ad..182333cea51 100644 --- a/crates/vm/src/builtins/int.rs +++ b/crates/vm/src/builtins/int.rs @@ -17,11 +17,11 @@ use crate::{ protocol::{PyNumberMethods, handle_bytes_to_int_err}, types::{AsNumber, Comparable, Constructor, Hashable, PyComparisonOp, Representable}, }; +use alloc::fmt; +use core::ops::{Neg, Not}; use malachite_bigint::{BigInt, Sign}; use num_integer::Integer; use num_traits::{One, Pow, PrimInt, Signed, ToPrimitive, Zero}; -use std::fmt; -use std::ops::{Neg, Not}; #[pyclass(module = false, name = "int")] #[derive(Debug)] @@ -289,7 +289,7 @@ impl PyInt { I::try_from(self.as_bigint()).map_err(|_| { vm.new_overflow_error(format!( "Python int too large to convert to Rust {}", - std::any::type_name::() + core::any::type_name::() )) }) } @@ -444,7 +444,7 @@ impl PyInt { #[pymethod] fn __sizeof__(&self) -> usize { - std::mem::size_of::() + (((self.value.bits() + 7) & !7) / 8) as usize + core::mem::size_of::() + (((self.value.bits() + 7) & !7) / 8) as usize } #[pymethod] diff --git a/crates/vm/src/builtins/list.rs b/crates/vm/src/builtins/list.rs index 12cab27a750..46145b339cf 100644 --- a/crates/vm/src/builtins/list.rs +++ b/crates/vm/src/builtins/list.rs @@ -20,7 +20,8 @@ use crate::{ utils::collection_repr, vm::VirtualMachine, }; -use std::{fmt, ops::DerefMut}; +use alloc::fmt; +use core::ops::DerefMut; #[pyclass(module = false, name = "list", unhashable = true, traverse)] #[derive(Default)] @@ -172,7 +173,7 @@ impl PyList { #[pymethod] fn clear(&self) { - let _removed = std::mem::take(self.borrow_vec_mut().deref_mut()); + let _removed = core::mem::take(self.borrow_vec_mut().deref_mut()); } #[pymethod] @@ -188,8 +189,8 @@ impl PyList { #[pymethod] fn __sizeof__(&self) -> usize { - std::mem::size_of::() - + self.elements.read().capacity() * std::mem::size_of::() + core::mem::size_of::() + + self.elements.read().capacity() * core::mem::size_of::() } #[pymethod] @@ -324,9 +325,9 @@ impl PyList { // replace list contents with [] for duration of sort. // this prevents keyfunc from messing with the list and makes it easy to // check if it tries to append elements to it. - let mut elements = std::mem::take(self.borrow_vec_mut().deref_mut()); + let mut elements = core::mem::take(self.borrow_vec_mut().deref_mut()); let res = do_sort(vm, &mut elements, options.key, options.reverse); - std::mem::swap(self.borrow_vec_mut().deref_mut(), &mut elements); + core::mem::swap(self.borrow_vec_mut().deref_mut(), &mut elements); res?; if !elements.is_empty() { @@ -375,7 +376,7 @@ impl MutObjectSequenceOp for PyList { inner.get(index).map(|r| r.as_ref()) } - fn do_lock(&self) -> impl std::ops::Deref { + fn do_lock(&self) -> impl core::ops::Deref { self.borrow_vec() } } @@ -397,7 +398,7 @@ impl Initializer for PyList { } else { vec![] }; - std::mem::swap(zelf.borrow_vec_mut().deref_mut(), &mut elements); + core::mem::swap(zelf.borrow_vec_mut().deref_mut(), &mut elements); Ok(()) } } diff --git a/crates/vm/src/builtins/map.rs b/crates/vm/src/builtins/map.rs index f5cee945ece..f83030824f1 100644 --- a/crates/vm/src/builtins/map.rs +++ b/crates/vm/src/builtins/map.rs @@ -42,7 +42,7 @@ impl PyMap { fn __length_hint__(&self, vm: &VirtualMachine) -> PyResult { self.iterators.iter().try_fold(0, |prev, cur| { let cur = cur.as_ref().to_owned().length_hint(0, vm)?; - let max = std::cmp::max(prev, cur); + let max = core::cmp::max(prev, cur); Ok(max) }) } diff --git a/crates/vm/src/builtins/memory.rs b/crates/vm/src/builtins/memory.rs index 4e895f92b7e..ff5df031c42 100644 --- a/crates/vm/src/builtins/memory.rs +++ b/crates/vm/src/builtins/memory.rs @@ -26,11 +26,11 @@ use crate::{ PyComparisonOp, Representable, SelfIter, }, }; +use core::{cmp::Ordering, fmt::Debug, mem::ManuallyDrop, ops::Range}; use crossbeam_utils::atomic::AtomicCell; use itertools::Itertools; use rustpython_common::lock::PyMutex; use std::sync::LazyLock; -use std::{cmp::Ordering, fmt::Debug, mem::ManuallyDrop, ops::Range}; #[derive(FromArgs)] pub struct PyMemoryViewNewArgs { diff --git a/crates/vm/src/builtins/module.rs b/crates/vm/src/builtins/module.rs index f8e42b28e0b..faa6e4813fd 100644 --- a/crates/vm/src/builtins/module.rs +++ b/crates/vm/src/builtins/module.rs @@ -32,8 +32,8 @@ pub struct PyModuleSlots { pub exec: Option, } -impl std::fmt::Debug for PyModuleSlots { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { +impl core::fmt::Debug for PyModuleSlots { + fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result { f.debug_struct("PyModuleSlots") .field("create", &self.create.is_some()) .field("exec", &self.exec.is_some()) diff --git a/crates/vm/src/builtins/object.rs b/crates/vm/src/builtins/object.rs index a61fb1e2971..e73f4b79ae0 100644 --- a/crates/vm/src/builtins/object.rs +++ b/crates/vm/src/builtins/object.rs @@ -218,7 +218,7 @@ fn object_getstate_default(obj: &PyObject, required: bool, vm: &VirtualMachine) // basicsize += std::mem::size_of::(); // } if let Some(ref slot_names) = slot_names { - basicsize += std::mem::size_of::() * slot_names.__len__(); + basicsize += core::mem::size_of::() * slot_names.__len__(); } if obj.class().slots.basicsize > basicsize { return Err( diff --git a/crates/vm/src/builtins/property.rs b/crates/vm/src/builtins/property.rs index 7ea36d39768..3a86867176a 100644 --- a/crates/vm/src/builtins/property.rs +++ b/crates/vm/src/builtins/property.rs @@ -10,7 +10,7 @@ use crate::{ function::{FuncArgs, PySetterValue}, types::{Constructor, GetDescriptor, Initializer}, }; -use std::sync::atomic::{AtomicBool, Ordering}; +use core::sync::atomic::{AtomicBool, Ordering}; #[pyclass(module = false, name = "property", traverse)] #[derive(Debug)] @@ -21,7 +21,7 @@ pub struct PyProperty { doc: PyRwLock>, name: PyRwLock>, #[pytraverse(skip)] - getter_doc: std::sync::atomic::AtomicBool, + getter_doc: core::sync::atomic::AtomicBool, } impl PyPayload for PyProperty { diff --git a/crates/vm/src/builtins/range.rs b/crates/vm/src/builtins/range.rs index 9f79f8efb2d..ab84c977ccd 100644 --- a/crates/vm/src/builtins/range.rs +++ b/crates/vm/src/builtins/range.rs @@ -14,11 +14,11 @@ use crate::{ Representable, SelfIter, }, }; +use core::cmp::max; use crossbeam_utils::atomic::AtomicCell; use malachite_bigint::{BigInt, Sign}; use num_integer::Integer; use num_traits::{One, Signed, ToPrimitive, Zero}; -use std::cmp::max; use std::sync::LazyLock; // Search flag passed to iter_search diff --git a/crates/vm/src/builtins/set.rs b/crates/vm/src/builtins/set.rs index 5582ff3323c..b1236e44e93 100644 --- a/crates/vm/src/builtins/set.rs +++ b/crates/vm/src/builtins/set.rs @@ -23,12 +23,13 @@ use crate::{ utils::collection_repr, vm::VirtualMachine, }; +use alloc::fmt; +use core::ops::Deref; use rustpython_common::{ atomic::{Ordering, PyAtomic, Radium}, hash, }; use std::sync::LazyLock; -use std::{fmt, ops::Deref}; pub type SetContentType = dict_inner::Dict<()>; @@ -50,7 +51,7 @@ impl PySet { fn fold_op( &self, - others: impl std::iter::Iterator, + others: impl core::iter::Iterator, op: fn(&PySetInner, ArgIterable, &VirtualMachine) -> PyResult, vm: &VirtualMachine, ) -> PyResult { @@ -68,7 +69,7 @@ impl PySet { Ok(Self { inner: self .inner - .fold_op(std::iter::once(other.into_iterable(vm)?), op, vm)?, + .fold_op(core::iter::once(other.into_iterable(vm)?), op, vm)?, }) } } @@ -111,7 +112,7 @@ impl PyFrozenSet { fn fold_op( &self, - others: impl std::iter::Iterator, + others: impl core::iter::Iterator, op: fn(&PySetInner, ArgIterable, &VirtualMachine) -> PyResult, vm: &VirtualMachine, ) -> PyResult { @@ -130,7 +131,7 @@ impl PyFrozenSet { Ok(Self { inner: self .inner - .fold_op(std::iter::once(other.into_iterable(vm)?), op, vm)?, + .fold_op(core::iter::once(other.into_iterable(vm)?), op, vm)?, ..Default::default() }) } @@ -191,7 +192,7 @@ impl PySetInner { fn fold_op( &self, - others: impl std::iter::Iterator, + others: impl core::iter::Iterator, op: fn(&Self, O, &VirtualMachine) -> PyResult, vm: &VirtualMachine, ) -> PyResult { @@ -352,7 +353,7 @@ impl PySetInner { fn update( &self, - others: impl std::iter::Iterator, + others: impl core::iter::Iterator, vm: &VirtualMachine, ) -> PyResult<()> { for iterable in others { @@ -395,7 +396,7 @@ impl PySetInner { fn intersection_update( &self, - others: impl std::iter::Iterator, + others: impl core::iter::Iterator, vm: &VirtualMachine, ) -> PyResult<()> { let temp_inner = self.fold_op(others, Self::intersection, vm)?; @@ -408,7 +409,7 @@ impl PySetInner { fn difference_update( &self, - others: impl std::iter::Iterator, + others: impl core::iter::Iterator, vm: &VirtualMachine, ) -> PyResult<()> { for iterable in others { @@ -422,7 +423,7 @@ impl PySetInner { fn symmetric_difference_update( &self, - others: impl std::iter::Iterator, + others: impl core::iter::Iterator, vm: &VirtualMachine, ) -> PyResult<()> { for iterable in others { @@ -539,7 +540,7 @@ impl PySet { #[pymethod] fn __sizeof__(&self) -> usize { - std::mem::size_of::() + self.inner.sizeof() + core::mem::size_of::() + self.inner.sizeof() } #[pymethod] @@ -731,7 +732,7 @@ impl PySet { #[pymethod] fn __iand__(zelf: PyRef, set: AnySet, vm: &VirtualMachine) -> PyResult> { zelf.inner - .intersection_update(std::iter::once(set.into_iterable(vm)?), vm)?; + .intersection_update(core::iter::once(set.into_iterable(vm)?), vm)?; Ok(zelf) } @@ -978,7 +979,7 @@ impl PyFrozenSet { #[pymethod] fn __sizeof__(&self) -> usize { - std::mem::size_of::() + self.inner.sizeof() + core::mem::size_of::() + self.inner.sizeof() } #[pymethod] @@ -1258,8 +1259,8 @@ impl AnySet { fn into_iterable_iter( self, vm: &VirtualMachine, - ) -> PyResult> { - Ok(std::iter::once(self.into_iterable(vm)?)) + ) -> PyResult> { + Ok(core::iter::once(self.into_iterable(vm)?)) } fn as_inner(&self) -> &PySetInner { diff --git a/crates/vm/src/builtins/str.rs b/crates/vm/src/builtins/str.rs index fa35c1725d4..0357e81b365 100644 --- a/crates/vm/src/builtins/str.rs +++ b/crates/vm/src/builtins/str.rs @@ -24,8 +24,10 @@ use crate::{ PyComparisonOp, Representable, SelfIter, }, }; +use alloc::{borrow::Cow, fmt}; use ascii::{AsciiChar, AsciiStr, AsciiString}; use bstr::ByteSlice; +use core::{char, mem, ops::Range}; use itertools::Itertools; use num_traits::ToPrimitive; use rustpython_common::{ @@ -37,8 +39,7 @@ use rustpython_common::{ str::DeduceStrKind, wtf8::{CodePoint, Wtf8, Wtf8Buf, Wtf8Chunk}, }; -use std::{borrow::Cow, char, fmt, ops::Range}; -use std::{mem, sync::LazyLock}; +use std::sync::LazyLock; use unic_ucd_bidi::BidiClass; use unic_ucd_category::GeneralCategory; use unic_ucd_ident::{is_xid_continue, is_xid_start}; @@ -191,8 +192,8 @@ impl From for PyStr { } } -impl<'a> From> for PyStr { - fn from(s: std::borrow::Cow<'a, str>) -> Self { +impl<'a> From> for PyStr { + fn from(s: alloc::borrow::Cow<'a, str>) -> Self { s.into_owned().into() } } @@ -632,7 +633,7 @@ impl PyStr { #[pymethod] fn __sizeof__(&self) -> usize { - std::mem::size_of::() + self.byte_len() * std::mem::size_of::() + core::mem::size_of::() + self.byte_len() * core::mem::size_of::() } #[pymethod(name = "__rmul__")] @@ -1045,7 +1046,7 @@ impl PyStr { #[pymethod] fn replace(&self, args: ReplaceArgs) -> Wtf8Buf { - use std::cmp::Ordering; + use core::cmp::Ordering; let s = self.as_wtf8(); let ReplaceArgs { old, new, count } = args; @@ -1361,7 +1362,7 @@ impl PyStr { let ch = bigint .as_bigint() .to_u32() - .and_then(std::char::from_u32) + .and_then(core::char::from_u32) .ok_or_else(|| { vm.new_value_error("character mapping must be in range(0x110000)") })?; @@ -1494,7 +1495,7 @@ impl PyRef { } struct CharLenStr<'a>(&'a str, usize); -impl std::ops::Deref for CharLenStr<'_> { +impl core::ops::Deref for CharLenStr<'_> { type Target = str; fn deref(&self) -> &Self::Target { @@ -1705,7 +1706,7 @@ pub struct FindArgs { } impl FindArgs { - fn get_value(self, len: usize) -> (PyStrRef, std::ops::Range) { + fn get_value(self, len: usize) -> (PyStrRef, core::ops::Range) { let range = adjust_indices(self.start, self.end, len); (self.sub, range) } @@ -1945,8 +1946,8 @@ impl PyPayload for PyUtf8Str { ctx.types.str_type } - fn payload_type_id() -> std::any::TypeId { - std::any::TypeId::of::() + fn payload_type_id() -> core::any::TypeId { + core::any::TypeId::of::() } fn validate_downcastable_from(obj: &PyObject) -> bool { @@ -2005,8 +2006,8 @@ impl From for PyUtf8Str { } } -impl<'a> From> for PyUtf8Str { - fn from(s: std::borrow::Cow<'a, str>) -> Self { +impl<'a> From> for PyUtf8Str { + fn from(s: alloc::borrow::Cow<'a, str>) -> Self { s.into_owned().into() } } @@ -2128,11 +2129,11 @@ impl AnyStr for str { Self::chars(self) } - fn get_bytes(&self, range: std::ops::Range) -> &Self { + fn get_bytes(&self, range: core::ops::Range) -> &Self { &self[range] } - fn get_chars(&self, range: std::ops::Range) -> &Self { + fn get_chars(&self, range: core::ops::Range) -> &Self { rustpython_common::str::get_chars(self, range) } @@ -2239,11 +2240,11 @@ impl AnyStr for Wtf8 { self.code_points() } - fn get_bytes(&self, range: std::ops::Range) -> &Self { + fn get_bytes(&self, range: core::ops::Range) -> &Self { &self[range] } - fn get_chars(&self, range: std::ops::Range) -> &Self { + fn get_chars(&self, range: core::ops::Range) -> &Self { rustpython_common::str::get_codepoints(self, range) } @@ -2361,11 +2362,11 @@ impl AnyStr for AsciiStr { self.chars() } - fn get_bytes(&self, range: std::ops::Range) -> &Self { + fn get_bytes(&self, range: core::ops::Range) -> &Self { &self[range] } - fn get_chars(&self, range: std::ops::Range) -> &Self { + fn get_chars(&self, range: core::ops::Range) -> &Self { &self[range] } @@ -2436,8 +2437,8 @@ impl PyStrInterned { } } -impl std::fmt::Display for PyStrInterned { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { +impl core::fmt::Display for PyStrInterned { + fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result { self.data.fmt(f) } } diff --git a/crates/vm/src/builtins/traceback.rs b/crates/vm/src/builtins/traceback.rs index 6bf4070c9b5..84493e7125f 100644 --- a/crates/vm/src/builtins/traceback.rs +++ b/crates/vm/src/builtins/traceback.rs @@ -81,7 +81,7 @@ impl Constructor for PyTraceback { impl PyTracebackRef { pub fn iter(&self) -> impl Iterator { - std::iter::successors(Some(self.clone()), |tb| tb.next.lock().clone()) + core::iter::successors(Some(self.clone()), |tb| tb.next.lock().clone()) } } diff --git a/crates/vm/src/builtins/tuple.rs b/crates/vm/src/builtins/tuple.rs index 13335428b35..70e4e20e405 100644 --- a/crates/vm/src/builtins/tuple.rs +++ b/crates/vm/src/builtins/tuple.rs @@ -21,7 +21,8 @@ use crate::{ utils::collection_repr, vm::VirtualMachine, }; -use std::{fmt, sync::LazyLock}; +use alloc::fmt; +use std::sync::LazyLock; #[pyclass(module = false, name = "tuple", traverse)] pub struct PyTuple { @@ -158,7 +159,7 @@ impl AsRef<[R]> for PyTuple { } } -impl std::ops::Deref for PyTuple { +impl core::ops::Deref for PyTuple { type Target = [R]; fn deref(&self) -> &[R] { @@ -166,18 +167,18 @@ impl std::ops::Deref for PyTuple { } } -impl<'a, R> std::iter::IntoIterator for &'a PyTuple { +impl<'a, R> core::iter::IntoIterator for &'a PyTuple { type Item = &'a R; - type IntoIter = std::slice::Iter<'a, R>; + type IntoIter = core::slice::Iter<'a, R>; fn into_iter(self) -> Self::IntoIter { self.iter() } } -impl<'a, R> std::iter::IntoIterator for &'a Py> { +impl<'a, R> core::iter::IntoIterator for &'a Py> { type Item = &'a R; - type IntoIter = std::slice::Iter<'a, R>; + type IntoIter = core::slice::Iter<'a, R>; fn into_iter(self) -> Self::IntoIter { self.iter() @@ -200,7 +201,7 @@ impl PyTuple { } #[inline] - pub fn iter(&self) -> std::slice::Iter<'_, R> { + pub fn iter(&self) -> core::slice::Iter<'_, R> { self.elements.iter() } } @@ -249,15 +250,15 @@ impl PyTuple> { // SAFETY: PyRef has the same layout as PyObjectRef unsafe { let elements: Vec = - std::mem::transmute::>, Vec>(elements); + core::mem::transmute::>, Vec>(elements); let tuple = PyTuple::::new_ref(elements, ctx); - std::mem::transmute::, PyRef>(tuple) + core::mem::transmute::, PyRef>(tuple) } } } #[pyclass( - itemsize = std::mem::size_of::(), + itemsize = core::mem::size_of::(), flags(BASETYPE, SEQUENCE, _MATCH_SELF), with(AsMapping, AsSequence, Hashable, Comparable, Iterable, Constructor, Representable) )] @@ -489,21 +490,21 @@ impl PyRef> { as TransmuteFromObject>::check(vm, elem)?; } // SAFETY: We just verified all elements are of type T - Ok(unsafe { std::mem::transmute::>>>(self) }) + Ok(unsafe { core::mem::transmute::>>>(self) }) } } impl PyRef>> { pub fn into_untyped(self) -> PyRef { // SAFETY: PyTuple> has the same layout as PyTuple - unsafe { std::mem::transmute::>(self) } + unsafe { core::mem::transmute::>(self) } } } impl Py>> { pub fn as_untyped(&self) -> &Py { // SAFETY: PyTuple> has the same layout as PyTuple - unsafe { std::mem::transmute::<&Self, &Py>(self) } + unsafe { core::mem::transmute::<&Self, &Py>(self) } } } diff --git a/crates/vm/src/builtins/type.rs b/crates/vm/src/builtins/type.rs index e807d3f4f8e..3eca5edc478 100644 --- a/crates/vm/src/builtins/type.rs +++ b/crates/vm/src/builtins/type.rs @@ -29,10 +29,11 @@ use crate::{ Representable, SLOT_DEFS, SetAttr, TypeDataRef, TypeDataRefMut, TypeDataSlot, }, }; +use core::{any::Any, borrow::Borrow, ops::Deref, pin::Pin, ptr::NonNull}; use indexmap::{IndexMap, map::Entry}; use itertools::Itertools; use num_traits::ToPrimitive; -use std::{any::Any, borrow::Borrow, collections::HashSet, ops::Deref, pin::Pin, ptr::NonNull}; +use std::collections::HashSet; #[pyclass(module = false, name = "type", traverse = "manual")] pub struct PyType { @@ -118,14 +119,14 @@ unsafe impl Traverse for PyAttributes { } } -impl std::fmt::Display for PyType { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - std::fmt::Display::fmt(&self.name(), f) +impl core::fmt::Display for PyType { + fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result { + core::fmt::Display::fmt(&self.name(), f) } } -impl std::fmt::Debug for PyType { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { +impl core::fmt::Debug for PyType { + fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result { write!(f, "[PyType {}]", &self.name()) } } @@ -368,7 +369,7 @@ impl PyType { // Inherit SEQUENCE and MAPPING flags from base class // For static types, we only have a single base - Self::inherit_patma_flags(&mut slots, std::slice::from_ref(&base)); + Self::inherit_patma_flags(&mut slots, core::slice::from_ref(&base)); if slots.basicsize == 0 { slots.basicsize = base.slots.basicsize; @@ -549,7 +550,7 @@ impl PyType { // Gather all members here: let mut attributes = PyAttributes::default(); - for bc in std::iter::once(self) + for bc in core::iter::once(self) .chain(self.mro.read().iter().map(|cls| -> &Self { cls })) .rev() { @@ -667,21 +668,21 @@ impl Py { where F: Fn(&Self) -> R, { - std::iter::once(self) + core::iter::once(self) .chain(self.mro.read().iter().map(|x| x.deref())) .map(f) .collect() } pub fn mro_collect(&self) -> Vec> { - std::iter::once(self) + core::iter::once(self) .chain(self.mro.read().iter().map(|x| x.deref())) .map(|x| x.to_owned()) .collect() } pub fn iter_base_chain(&self) -> impl Iterator { - std::iter::successors(Some(self), |cls| cls.base.as_deref()) + core::iter::successors(Some(self), |cls| cls.base.as_deref()) } pub fn extend_methods(&'static self, method_defs: &'static [PyMethodDef], ctx: &Context) { @@ -846,7 +847,7 @@ impl PyType { // then drop the old value after releasing the lock let _old_qualname = { let mut qualname_guard = heap_type.qualname.write(); - std::mem::replace(&mut *qualname_guard, str_value) + core::mem::replace(&mut *qualname_guard, str_value) }; // old_qualname is dropped here, outside the lock scope @@ -1012,7 +1013,7 @@ impl PyType { // then drop the old value after releasing the lock (similar to CPython's Py_SETREF) let _old_name = { let mut name_guard = self.heaptype_ext.as_ref().unwrap().name.write(); - std::mem::replace(&mut *name_guard, name) + core::mem::replace(&mut *name_guard, name) }; // old_name is dropped here, outside the lock scope diff --git a/crates/vm/src/builtins/union.rs b/crates/vm/src/builtins/union.rs index 83e1316d027..0342442b83d 100644 --- a/crates/vm/src/builtins/union.rs +++ b/crates/vm/src/builtins/union.rs @@ -11,7 +11,7 @@ use crate::{ stdlib::typing::TypeAliasType, types::{AsMapping, AsNumber, Comparable, GetAttr, Hashable, PyComparisonOp, Representable}, }; -use std::fmt; +use alloc::fmt; use std::sync::LazyLock; const CLS_ATTRS: &[&str] = &["__module__"]; diff --git a/crates/vm/src/bytes_inner.rs b/crates/vm/src/bytes_inner.rs index 8593f16fcd9..bb5db442c35 100644 --- a/crates/vm/src/bytes_inner.rs +++ b/crates/vm/src/bytes_inner.rs @@ -151,7 +151,7 @@ impl ByteInnerFindOptions { self, len: usize, vm: &VirtualMachine, - ) -> PyResult<(Vec, std::ops::Range)> { + ) -> PyResult<(Vec, core::ops::Range)> { let sub = match self.sub { Either::A(v) => v.elements.to_vec(), Either::B(int) => vec![int.as_bigint().byte_or(vm)?], @@ -719,7 +719,7 @@ impl PyBytesInner { // len(self)>=1, from="", len(to)>=1, max_count>=1 fn replace_interleave(&self, to: Self, max_count: Option) -> Vec { let place_count = self.elements.len() + 1; - let count = max_count.map_or(place_count, |v| std::cmp::min(v, place_count)) - 1; + let count = max_count.map_or(place_count, |v| core::cmp::min(v, place_count)) - 1; let capacity = self.elements.len() + count * to.len(); let mut result = Vec::with_capacity(capacity); let to_slice = to.elements.as_slice(); @@ -952,7 +952,7 @@ where fn count_substring(haystack: &[u8], needle: &[u8], max_count: Option) -> usize { let substrings = haystack.find_iter(needle); if let Some(max_count) = max_count { - std::cmp::min(substrings.take(max_count).count(), max_count) + core::cmp::min(substrings.take(max_count).count(), max_count) } else { substrings.count() } @@ -1025,11 +1025,11 @@ impl AnyStr for [u8] { self.iter().copied() } - fn get_bytes(&self, range: std::ops::Range) -> &Self { + fn get_bytes(&self, range: core::ops::Range) -> &Self { &self[range] } - fn get_chars(&self, range: std::ops::Range) -> &Self { + fn get_chars(&self, range: core::ops::Range) -> &Self { &self[range] } @@ -1120,7 +1120,7 @@ fn hex_impl(bytes: &[u8], sep: u8, bytes_per_sep: isize) -> String { let len = bytes.len(); let buf = if bytes_per_sep < 0 { - let bytes_per_sep = std::cmp::min(len, (-bytes_per_sep) as usize); + let bytes_per_sep = core::cmp::min(len, (-bytes_per_sep) as usize); let chunks = (len - 1) / bytes_per_sep; let chunked = chunks * bytes_per_sep; let unchunked = len - chunked; @@ -1139,7 +1139,7 @@ fn hex_impl(bytes: &[u8], sep: u8, bytes_per_sep: isize) -> String { hex::encode_to_slice(&bytes[chunked..], &mut buf[j..j + unchunked * 2]).unwrap(); buf } else { - let bytes_per_sep = std::cmp::min(len, bytes_per_sep as usize); + let bytes_per_sep = core::cmp::min(len, bytes_per_sep as usize); let chunks = (len - 1) / bytes_per_sep; let chunked = chunks * bytes_per_sep; let unchunked = len - chunked; diff --git a/crates/vm/src/cformat.rs b/crates/vm/src/cformat.rs index efb3cb2acc9..939b1c7760f 100644 --- a/crates/vm/src/cformat.rs +++ b/crates/vm/src/cformat.rs @@ -342,7 +342,7 @@ pub(crate) fn cformat_bytes( let values = if let Some(tup) = values_obj.downcast_ref::() { tup.as_slice() } else { - std::slice::from_ref(&values_obj) + core::slice::from_ref(&values_obj) }; let mut value_iter = values.iter(); @@ -435,7 +435,7 @@ pub(crate) fn cformat_string( let values = if let Some(tup) = values_obj.downcast_ref::() { tup.as_slice() } else { - std::slice::from_ref(&values_obj) + core::slice::from_ref(&values_obj) }; let mut value_iter = values.iter(); diff --git a/crates/vm/src/class.rs b/crates/vm/src/class.rs index ce41abcc60b..98dc6fd2ed2 100644 --- a/crates/vm/src/class.rs +++ b/crates/vm/src/class.rs @@ -169,7 +169,7 @@ pub trait PyClassImpl: PyClassDef { // Exception: object itself should have __new__ in its dict if let Some(slot_new) = class.slots.new.load() { let object_new = ctx.types.object_type.slots.new.load(); - let is_object_itself = std::ptr::eq(class, ctx.types.object_type); + let is_object_itself = core::ptr::eq(class, ctx.types.object_type); let is_inherited_from_object = !is_object_itself && object_new.is_some_and(|obj_new| slot_new as usize == obj_new as usize); @@ -203,7 +203,7 @@ pub trait PyClassImpl: PyClassDef { Self::extend_class(ctx, unsafe { // typ will be saved in static_cell let r: &Py = &typ; - let r: &'static Py = std::mem::transmute(r); + let r: &'static Py = core::mem::transmute(r); r }); typ diff --git a/crates/vm/src/codecs.rs b/crates/vm/src/codecs.rs index 2edb67b497b..3241dee4981 100644 --- a/crates/vm/src/codecs.rs +++ b/crates/vm/src/codecs.rs @@ -16,12 +16,10 @@ use crate::{ convert::ToPyObject, function::{ArgBytesLike, PyMethodDef}, }; +use alloc::borrow::Cow; +use core::ops::{self, Range}; use once_cell::unsync::OnceCell; -use std::{ - borrow::Cow, - collections::HashMap, - ops::{self, Range}, -}; +use std::collections::HashMap; pub struct CodecsRegistry { inner: PyRwLock, diff --git a/crates/vm/src/convert/try_from.rs b/crates/vm/src/convert/try_from.rs index 4f921e9c5de..ceb7d003e9b 100644 --- a/crates/vm/src/convert/try_from.rs +++ b/crates/vm/src/convert/try_from.rs @@ -122,7 +122,7 @@ impl<'a, T: PyPayload> TryFromBorrowedObject<'a> for &'a Py { } } -impl TryFromObject for std::time::Duration { +impl TryFromObject for core::time::Duration { fn try_from_object(vm: &VirtualMachine, obj: PyObjectRef) -> PyResult { if let Some(float) = obj.downcast_ref::() { let f = float.to_f64(); diff --git a/crates/vm/src/dict_inner.rs b/crates/vm/src/dict_inner.rs index 02e237afb00..d57f8be0fe7 100644 --- a/crates/vm/src/dict_inner.rs +++ b/crates/vm/src/dict_inner.rs @@ -16,8 +16,9 @@ use crate::{ }, object::{Traverse, TraverseFn}, }; +use alloc::fmt; +use core::{mem::size_of, ops::ControlFlow}; use num_traits::ToPrimitive; -use std::{fmt, mem::size_of, ops::ControlFlow}; // HashIndex is intended to be same size with hash::PyHash // but it doesn't mean the values are compatible with actual PyHash value @@ -281,7 +282,7 @@ impl Dict { continue; }; if entry.index == index_index { - let removed = std::mem::replace(&mut entry.value, value); + let removed = core::mem::replace(&mut entry.value, value); // defer dec RC break Some(removed); } else { @@ -357,7 +358,7 @@ impl Dict { inner.used = 0; inner.filled = 0; // defer dec rc - std::mem::take(&mut inner.entries) + core::mem::take(&mut inner.entries) }; } @@ -633,7 +634,7 @@ impl Dict { // returns Err(()) if changed since lookup fn pop_inner(&self, lookup: LookupResult) -> PopInnerResult { - self.pop_inner_if(lookup, |_| Ok::<_, std::convert::Infallible>(true)) + self.pop_inner_if(lookup, |_| Ok::<_, core::convert::Infallible>(true)) .unwrap_or_else(|x| match x {}) } diff --git a/crates/vm/src/exceptions.rs b/crates/vm/src/exceptions.rs index 4e8b572e457..b1a654d58c4 100644 --- a/crates/vm/src/exceptions.rs +++ b/crates/vm/src/exceptions.rs @@ -33,8 +33,8 @@ unsafe impl Traverse for PyBaseException { } } -impl std::fmt::Debug for PyBaseException { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { +impl core::fmt::Debug for PyBaseException { + fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result { // TODO: implement more detailed, non-recursive Debug formatter f.write_str("PyBaseException") } @@ -1173,7 +1173,7 @@ pub fn cstring_error(vm: &VirtualMachine) -> PyBaseExceptionRef { vm.new_value_error("embedded null character") } -impl ToPyException for std::ffi::NulError { +impl ToPyException for alloc::ffi::NulError { fn to_pyexception(&self, vm: &VirtualMachine) -> PyBaseExceptionRef { cstring_error(vm) } @@ -1604,8 +1604,8 @@ pub(super) mod types { } } - impl std::fmt::Debug for PyOSError { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + impl core::fmt::Debug for PyOSError { + fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result { f.debug_struct("PyOSError").finish_non_exhaustive() } } diff --git a/crates/vm/src/frame.rs b/crates/vm/src/frame.rs index 3f470e5453f..6acf0e84795 100644 --- a/crates/vm/src/frame.rs +++ b/crates/vm/src/frame.rs @@ -18,13 +18,14 @@ use crate::{ types::PyTypeFlags, vm::{Context, PyMethod}, }; +use alloc::fmt; +use core::iter::zip; +#[cfg(feature = "threading")] +use core::sync::atomic; use indexmap::IndexMap; use itertools::Itertools; use rustpython_common::{boxvec::BoxVec, lock::PyMutex, wtf8::Wtf8Buf}; use rustpython_compiler_core::SourceLocation; -#[cfg(feature = "threading")] -use std::sync::atomic; -use std::{fmt, iter::zip}; #[derive(Clone, Debug)] struct Block { @@ -91,7 +92,7 @@ struct FrameState { #[cfg(feature = "threading")] type Lasti = atomic::AtomicU32; #[cfg(not(feature = "threading"))] -type Lasti = std::cell::Cell; +type Lasti = core::cell::Cell; #[pyclass(module = false, name = "frame")] pub struct Frame { @@ -142,7 +143,7 @@ impl Frame { func_obj: Option, vm: &VirtualMachine, ) -> Self { - let cells_frees = std::iter::repeat_with(|| PyCell::default().into_ref(&vm.ctx)) + let cells_frees = core::iter::repeat_with(|| PyCell::default().into_ref(&vm.ctx)) .take(code.cellvars.len()) .chain(closure.iter().cloned()) .collect(); @@ -189,7 +190,7 @@ impl Frame { let locals = &self.locals; let code = &**self.code; let map = &code.varnames; - let j = std::cmp::min(map.len(), code.varnames.len()); + let j = core::cmp::min(map.len(), code.varnames.len()); if !code.varnames.is_empty() { let fastlocals = self.fastlocals.lock(); for (&k, v) in zip(&map[..j], &**fastlocals) { @@ -2243,14 +2244,14 @@ impl ExecutingFrame<'_> { } })?; let msg = match elements.len().cmp(&(size as usize)) { - std::cmp::Ordering::Equal => { + core::cmp::Ordering::Equal => { self.state.stack.extend(elements.into_iter().rev()); return Ok(None); } - std::cmp::Ordering::Greater => { + core::cmp::Ordering::Greater => { format!("too many values to unpack (expected {size})") } - std::cmp::Ordering::Less => format!( + core::cmp::Ordering::Less => format!( "not enough values to unpack (expected {}, got {})", size, elements.len() @@ -2525,7 +2526,7 @@ impl ExecutingFrame<'_> { #[inline] fn replace_top(&mut self, mut top: PyObjectRef) -> PyObjectRef { let last = self.state.stack.last_mut().unwrap(); - std::mem::swap(&mut top, last); + core::mem::swap(&mut top, last); top } @@ -2561,12 +2562,12 @@ impl fmt::Debug for Frame { if elem.downcastable::() { s.push_str("\n > {frame}"); } else { - std::fmt::write(&mut s, format_args!("\n > {elem:?}")).unwrap(); + core::fmt::write(&mut s, format_args!("\n > {elem:?}")).unwrap(); } s }); let block_str = state.blocks.iter().fold(String::new(), |mut s, elem| { - std::fmt::write(&mut s, format_args!("\n > {elem:?}")).unwrap(); + core::fmt::write(&mut s, format_args!("\n > {elem:?}")).unwrap(); s }); // TODO: fix this up diff --git a/crates/vm/src/function/argument.rs b/crates/vm/src/function/argument.rs index d657ff6be8f..a4877cf4042 100644 --- a/crates/vm/src/function/argument.rs +++ b/crates/vm/src/function/argument.rs @@ -4,9 +4,9 @@ use crate::{ convert::ToPyObject, object::{Traverse, TraverseFn}, }; +use core::ops::RangeInclusive; use indexmap::IndexMap; use itertools::Itertools; -use std::ops::RangeInclusive; pub trait IntoFuncArgs: Sized { fn into_args(self, vm: &VirtualMachine) -> FuncArgs; @@ -100,7 +100,7 @@ impl From for FuncArgs { impl FromArgs for FuncArgs { fn from_args(_vm: &VirtualMachine, args: &mut FuncArgs) -> Result { - Ok(std::mem::take(args)) + Ok(core::mem::take(args)) } } @@ -424,7 +424,7 @@ impl PosArgs { self.0 } - pub fn iter(&self) -> std::slice::Iter<'_, T> { + pub fn iter(&self) -> core::slice::Iter<'_, T> { self.0.iter() } } @@ -469,7 +469,7 @@ where impl IntoIterator for PosArgs { type Item = T; - type IntoIter = std::vec::IntoIter; + type IntoIter = alloc::vec::IntoIter; fn into_iter(self) -> Self::IntoIter { self.0.into_iter() diff --git a/crates/vm/src/function/builtin.rs b/crates/vm/src/function/builtin.rs index 1a91e4344ba..06fd6a44f54 100644 --- a/crates/vm/src/function/builtin.rs +++ b/crates/vm/src/function/builtin.rs @@ -3,7 +3,7 @@ use crate::{ Py, PyPayload, PyRef, PyResult, VirtualMachine, convert::ToPyResult, object::PyThreadingConstraint, }; -use std::marker::PhantomData; +use core::marker::PhantomData; /// A built-in Python function. // PyCFunction in CPython @@ -54,14 +54,14 @@ const fn zst_ref_out_of_thin_air(x: T) -> &'static T { // if T is zero-sized, there's no issue forgetting it - even if it does have a Drop impl, it // would never get called anyway if we consider this semantically a Box::leak(Box::new(x))-type // operation. if T isn't zero-sized, we don't have to worry about it because we'll fail to compile. - std::mem::forget(x); + core::mem::forget(x); const { - if std::mem::size_of::() != 0 { + if core::mem::size_of::() != 0 { panic!("can't use a non-zero-sized type here") } // SAFETY: we just confirmed that T is zero-sized, so we can // pull a value of it out of thin air. - unsafe { std::ptr::NonNull::::dangling().as_ref() } + unsafe { core::ptr::NonNull::::dangling().as_ref() } } } diff --git a/crates/vm/src/function/either.rs b/crates/vm/src/function/either.rs index 8700c6150db..9ee7f028bd2 100644 --- a/crates/vm/src/function/either.rs +++ b/crates/vm/src/function/either.rs @@ -1,7 +1,7 @@ use crate::{ AsObject, PyObject, PyObjectRef, PyResult, TryFromObject, VirtualMachine, convert::ToPyObject, }; -use std::borrow::Borrow; +use core::borrow::Borrow; pub enum Either { A(A), diff --git a/crates/vm/src/function/fspath.rs b/crates/vm/src/function/fspath.rs index 44d41ab7632..7d3a0dcbbd5 100644 --- a/crates/vm/src/function/fspath.rs +++ b/crates/vm/src/function/fspath.rs @@ -5,7 +5,8 @@ use crate::{ function::PyStr, protocol::PyBuffer, }; -use std::{borrow::Cow, ffi::OsStr, path::PathBuf}; +use alloc::borrow::Cow; +use std::{ffi::OsStr, path::PathBuf}; /// Helper to implement os.fspath() #[derive(Clone)] @@ -111,8 +112,8 @@ impl FsPath { Ok(path) } - pub fn to_cstring(&self, vm: &VirtualMachine) -> PyResult { - std::ffi::CString::new(self.as_bytes()).map_err(|e| e.into_pyexception(vm)) + pub fn to_cstring(&self, vm: &VirtualMachine) -> PyResult { + alloc::ffi::CString::new(self.as_bytes()).map_err(|e| e.into_pyexception(vm)) } #[cfg(windows)] diff --git a/crates/vm/src/function/method.rs b/crates/vm/src/function/method.rs index 5e109176c5e..6440fd801fc 100644 --- a/crates/vm/src/function/method.rs +++ b/crates/vm/src/function/method.rs @@ -251,14 +251,14 @@ impl PyMethodDef { } } -impl std::fmt::Debug for PyMethodDef { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { +impl core::fmt::Debug for PyMethodDef { + fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result { f.debug_struct("PyMethodDef") .field("name", &self.name) .field( "func", &(unsafe { - std::mem::transmute::<&dyn PyNativeFn, [usize; 2]>(self.func)[1] as *const u8 + core::mem::transmute::<&dyn PyNativeFn, [usize; 2]>(self.func)[1] as *const u8 }), ) .field("flags", &self.flags) diff --git a/crates/vm/src/function/number.rs b/crates/vm/src/function/number.rs index 7bb37b8f549..fb872cc48fd 100644 --- a/crates/vm/src/function/number.rs +++ b/crates/vm/src/function/number.rs @@ -1,9 +1,9 @@ use super::argument::OptionalArg; use crate::{AsObject, PyObjectRef, PyResult, TryFromObject, VirtualMachine, builtins::PyIntRef}; +use core::ops::Deref; use malachite_bigint::BigInt; use num_complex::Complex64; use num_traits::PrimInt; -use std::ops::Deref; /// A Python complex-like object. /// @@ -62,7 +62,7 @@ pub struct ArgIntoFloat { impl ArgIntoFloat { pub fn vec_into_f64(v: Vec) -> Vec { // TODO: Vec::into_raw_parts once stabilized - let mut v = std::mem::ManuallyDrop::new(v); + let mut v = core::mem::ManuallyDrop::new(v); let (p, l, c) = (v.as_mut_ptr(), v.len(), v.capacity()); // SAFETY: IntoPyFloat is repr(transparent) over f64 unsafe { Vec::from_raw_parts(p.cast(), l, c) } diff --git a/crates/vm/src/function/protocol.rs b/crates/vm/src/function/protocol.rs index a87ef339edd..94bdd3027eb 100644 --- a/crates/vm/src/function/protocol.rs +++ b/crates/vm/src/function/protocol.rs @@ -7,7 +7,7 @@ use crate::{ protocol::{PyIter, PyIterIter, PyMapping}, types::GenericMethod, }; -use std::{borrow::Borrow, marker::PhantomData, ops::Deref}; +use core::{borrow::Borrow, marker::PhantomData, ops::Deref}; #[derive(Clone, Traverse)] pub struct ArgCallable { @@ -24,8 +24,8 @@ impl ArgCallable { } } -impl std::fmt::Debug for ArgCallable { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { +impl core::fmt::Debug for ArgCallable { + fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result { f.debug_struct("ArgCallable") .field("obj", &self.obj) .field("call", &format!("{:08x}", self.call as usize)) @@ -203,7 +203,7 @@ impl ArgSequence { } } -impl std::ops::Deref for ArgSequence { +impl core::ops::Deref for ArgSequence { type Target = [T]; #[inline(always)] fn deref(&self) -> &[T] { @@ -213,14 +213,14 @@ impl std::ops::Deref for ArgSequence { impl<'a, T> IntoIterator for &'a ArgSequence { type Item = &'a T; - type IntoIter = std::slice::Iter<'a, T>; + type IntoIter = core::slice::Iter<'a, T>; fn into_iter(self) -> Self::IntoIter { self.iter() } } impl IntoIterator for ArgSequence { type Item = T; - type IntoIter = std::vec::IntoIter; + type IntoIter = alloc::vec::IntoIter; fn into_iter(self) -> Self::IntoIter { self.0.into_iter() } diff --git a/crates/vm/src/intern.rs b/crates/vm/src/intern.rs index a5b2a798d53..a50b8871cb9 100644 --- a/crates/vm/src/intern.rs +++ b/crates/vm/src/intern.rs @@ -6,10 +6,8 @@ use crate::{ common::lock::PyRwLock, convert::ToPyObject, }; -use std::{ - borrow::{Borrow, ToOwned}, - ops::Deref, -}; +use alloc::borrow::ToOwned; +use core::{borrow::Borrow, ops::Deref}; #[derive(Debug)] pub struct StringPool { @@ -86,8 +84,8 @@ pub struct CachedPyStrRef { inner: PyRefExact, } -impl std::hash::Hash for CachedPyStrRef { - fn hash(&self, state: &mut H) { +impl core::hash::Hash for CachedPyStrRef { + fn hash(&self, state: &mut H) { self.inner.as_wtf8().hash(state) } } @@ -100,7 +98,7 @@ impl PartialEq for CachedPyStrRef { impl Eq for CachedPyStrRef {} -impl std::borrow::Borrow for CachedPyStrRef { +impl core::borrow::Borrow for CachedPyStrRef { #[inline] fn borrow(&self) -> &Wtf8 { self.as_wtf8() @@ -119,7 +117,7 @@ impl CachedPyStrRef { /// the given cache must be alive while returned reference is alive #[inline] const unsafe fn as_interned_str(&self) -> &'static PyStrInterned { - unsafe { std::mem::transmute_copy(self) } + unsafe { core::mem::transmute_copy(self) } } #[inline] @@ -135,7 +133,7 @@ pub struct PyInterned { impl PyInterned { #[inline] pub fn leak(cache: PyRef) -> &'static Self { - unsafe { std::mem::transmute(cache) } + unsafe { core::mem::transmute(cache) } } #[inline] @@ -163,9 +161,9 @@ impl Borrow for PyInterned { // NOTE: std::hash::Hash of Self and Self::Borrowed *must* be the same // This is ok only because PyObject doesn't implement Hash -impl std::hash::Hash for PyInterned { +impl core::hash::Hash for PyInterned { #[inline(always)] - fn hash(&self, state: &mut H) { + fn hash(&self, state: &mut H) { self.get_id().hash(state) } } @@ -188,15 +186,15 @@ impl Deref for PyInterned { impl PartialEq for PyInterned { #[inline(always)] fn eq(&self, other: &Self) -> bool { - std::ptr::eq(self, other) + core::ptr::eq(self, other) } } impl Eq for PyInterned {} -impl std::fmt::Debug for PyInterned { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - std::fmt::Debug::fmt(&**self, f)?; +impl core::fmt::Debug for PyInterned { + fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result { + core::fmt::Debug::fmt(&**self, f)?; write!(f, "@{:p}", self.as_ptr()) } } @@ -308,7 +306,7 @@ impl MaybeInternedString for Py { #[inline(always)] fn as_interned(&self) -> Option<&'static PyStrInterned> { if self.as_object().is_interned() { - Some(unsafe { std::mem::transmute::<&Self, &PyInterned>(self) }) + Some(unsafe { core::mem::transmute::<&Self, &PyInterned>(self) }) } else { None } diff --git a/crates/vm/src/lib.rs b/crates/vm/src/lib.rs index f461c612955..3f0eee278a2 100644 --- a/crates/vm/src/lib.rs +++ b/crates/vm/src/lib.rs @@ -24,6 +24,7 @@ extern crate bitflags; #[macro_use] extern crate log; // extern crate env_logger; +extern crate alloc; #[macro_use] extern crate rustpython_derive; diff --git a/crates/vm/src/macros.rs b/crates/vm/src/macros.rs index 1284c202782..f5c912d89cf 100644 --- a/crates/vm/src/macros.rs +++ b/crates/vm/src/macros.rs @@ -146,8 +146,8 @@ macro_rules! match_class { }; (match ($obj:expr) { ref $binding:ident @ $class:ty => $expr:expr, $($rest:tt)* }) => { match $obj.downcast_ref::<$class>() { - ::std::option::Option::Some($binding) => $expr, - ::std::option::Option::None => $crate::match_class!(match ($obj) { $($rest)* }), + core::option::Option::Some($binding) => $expr, + core::option::Option::None => $crate::match_class!(match ($obj) { $($rest)* }), } }; diff --git a/crates/vm/src/object/core.rs b/crates/vm/src/object/core.rs index d52a33884ce..cee3fac266e 100644 --- a/crates/vm/src/object/core.rs +++ b/crates/vm/src/object/core.rs @@ -31,11 +31,13 @@ use crate::{ object::traverse::{MaybeTraverse, Traverse, TraverseFn}, }; use itertools::Itertools; -use std::{ + +use alloc::fmt; + +use core::{ any::TypeId, borrow::Borrow, cell::UnsafeCell, - fmt, marker::PhantomData, mem::ManuallyDrop, ops::Deref, @@ -82,7 +84,7 @@ pub(super) struct Erased; pub(super) unsafe fn drop_dealloc_obj(x: *mut PyObject) { drop(unsafe { Box::from_raw(x as *mut PyInner) }); } -pub(super) unsafe fn debug_obj( +pub(super) unsafe fn debug_obj( x: &PyObject, f: &mut fmt::Formatter<'_>, ) -> fmt::Result { @@ -114,7 +116,7 @@ pub(super) struct PyInner { pub(super) payload: T, } -pub(crate) const SIZEOF_PYOBJECT_HEAD: usize = std::mem::size_of::>(); +pub(crate) const SIZEOF_PYOBJECT_HEAD: usize = core::mem::size_of::>(); impl fmt::Debug for PyInner { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { @@ -376,7 +378,7 @@ impl PyWeak { let node_ptr = unsafe { NonNull::new_unchecked(py_inner as *mut Py) }; // the list doesn't have ownership over its PyRef! we're being dropped // right now so that should be obvious!! - std::mem::forget(unsafe { guard.list.remove(node_ptr) }); + core::mem::forget(unsafe { guard.list.remove(node_ptr) }); guard.ref_count -= 1; if Some(node_ptr) == guard.generic_weakref { guard.generic_weakref = None; @@ -438,11 +440,11 @@ impl InstanceDict { #[inline] pub fn replace(&self, d: PyDictRef) -> PyDictRef { - std::mem::replace(&mut self.d.write(), d) + core::mem::replace(&mut self.d.write(), d) } } -impl PyInner { +impl PyInner { fn new(payload: T, typ: PyTypeRef, dict: Option) -> Box { let member_count = typ.slots.member_count; Box::new(Self { @@ -453,7 +455,7 @@ impl PyInner { dict: dict.map(InstanceDict::new), weak_list: WeakRefList::new(), payload, - slots: std::iter::repeat_with(|| PyRwLock::new(None)) + slots: core::iter::repeat_with(|| PyRwLock::new(None)) .take(member_count) .collect_vec() .into_boxed_slice(), @@ -513,7 +515,7 @@ impl PyObjectRef { #[inline(always)] pub const fn into_raw(self) -> NonNull { let ptr = self.ptr; - std::mem::forget(self); + core::mem::forget(self); ptr } @@ -946,12 +948,12 @@ impl Borrow for Py { } } -impl std::hash::Hash for Py +impl core::hash::Hash for Py where - T: std::hash::Hash + PyPayload, + T: core::hash::Hash + PyPayload, { #[inline] - fn hash(&self, state: &mut H) { + fn hash(&self, state: &mut H) { self.deref().hash(state) } } @@ -978,7 +980,7 @@ where } } -impl fmt::Debug for Py { +impl fmt::Debug for Py { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { (**self).fmt(f) } @@ -1059,12 +1061,12 @@ impl PyRef { pub const fn leak(pyref: Self) -> &'static Py { let ptr = pyref.ptr; - std::mem::forget(pyref); + core::mem::forget(pyref); unsafe { ptr.as_ref() } } } -impl PyRef { +impl PyRef { #[inline(always)] pub fn new_ref(payload: T, typ: crate::builtins::PyTypeRef, dict: Option) -> Self { let inner = Box::into_raw(PyInner::new(payload, typ, dict)); @@ -1074,9 +1076,9 @@ impl PyRef { } } -impl PyRef +impl PyRef where - T::Base: std::fmt::Debug, + T::Base: core::fmt::Debug, { /// Converts this reference to the base type (ownership transfer). /// # Safety @@ -1086,7 +1088,7 @@ where let obj: PyObjectRef = self.into(); match obj.downcast() { Ok(base_ref) => base_ref, - Err(_) => unsafe { std::hint::unreachable_unchecked() }, + Err(_) => unsafe { core::hint::unreachable_unchecked() }, } } #[inline] @@ -1098,7 +1100,7 @@ where let obj: PyObjectRef = self.into(); match obj.downcast::() { Ok(upcast_ref) => upcast_ref, - Err(_) => unsafe { std::hint::unreachable_unchecked() }, + Err(_) => unsafe { core::hint::unreachable_unchecked() }, } } } @@ -1176,12 +1178,12 @@ impl Deref for PyRef { } } -impl std::hash::Hash for PyRef +impl core::hash::Hash for PyRef where - T: std::hash::Hash + PyPayload, + T: core::hash::Hash + PyPayload, { #[inline] - fn hash(&self, state: &mut H) { + fn hash(&self, state: &mut H) { self.deref().hash(state) } } @@ -1230,10 +1232,10 @@ macro_rules! partially_init { $($uninit_field: unreachable!(),)* }}; } - let mut m = ::std::mem::MaybeUninit::<$ty>::uninit(); + let mut m = ::core::mem::MaybeUninit::<$ty>::uninit(); #[allow(unused_unsafe)] unsafe { - $(::std::ptr::write(&mut (*m.as_mut_ptr()).$init_field, $init_value);)* + $(::core::ptr::write(&mut (*m.as_mut_ptr()).$init_field, $init_value);)* } m }}; @@ -1241,7 +1243,7 @@ macro_rules! partially_init { pub(crate) fn init_type_hierarchy() -> (PyTypeRef, PyTypeRef, PyTypeRef) { use crate::{builtins::object, class::PyClassImpl}; - use std::mem::MaybeUninit; + use core::mem::MaybeUninit; // `type` inherits from `object` // and both `type` and `object are instances of `type`. diff --git a/crates/vm/src/object/ext.rs b/crates/vm/src/object/ext.rs index 88f5fdc66d7..c1a5f63f85e 100644 --- a/crates/vm/src/object/ext.rs +++ b/crates/vm/src/object/ext.rs @@ -12,9 +12,10 @@ use crate::{ convert::{IntoPyException, ToPyObject, ToPyResult, TryFromObject}, vm::Context, }; -use std::{ +use alloc::fmt; + +use core::{ borrow::Borrow, - fmt, marker::PhantomData, ops::Deref, ptr::{NonNull, null_mut}, @@ -108,7 +109,7 @@ impl AsRef> for PyExact { } } -impl std::borrow::ToOwned for PyExact { +impl alloc::borrow::ToOwned for PyExact { type Owned = PyRefExact; fn to_owned(&self) -> Self::Owned { @@ -581,7 +582,7 @@ impl ToPyObject for &PyObject { // explicitly implementing `ToPyObject`. impl ToPyObject for T where - T: PyPayload + std::fmt::Debug + Sized, + T: PyPayload + core::fmt::Debug + Sized, { #[inline(always)] fn to_pyobject(self, vm: &VirtualMachine) -> PyObjectRef { diff --git a/crates/vm/src/object/payload.rs b/crates/vm/src/object/payload.rs index 4b900b7caa1..3a2f42675f7 100644 --- a/crates/vm/src/object/payload.rs +++ b/crates/vm/src/object/payload.rs @@ -27,8 +27,8 @@ pub(crate) fn cold_downcast_type_error( pub trait PyPayload: MaybeTraverse + PyThreadingConstraint + Sized + 'static { #[inline] - fn payload_type_id() -> std::any::TypeId { - std::any::TypeId::of::() + fn payload_type_id() -> core::any::TypeId { + core::any::TypeId::of::() } /// # Safety: this function should only be called if `payload_type_id` matches the type of `obj`. @@ -56,7 +56,7 @@ pub trait PyPayload: MaybeTraverse + PyThreadingConstraint + Sized + 'static { #[inline] fn into_pyobject(self, vm: &VirtualMachine) -> PyObjectRef where - Self: std::fmt::Debug, + Self: core::fmt::Debug, { self.into_ref(&vm.ctx).into() } @@ -64,7 +64,7 @@ pub trait PyPayload: MaybeTraverse + PyThreadingConstraint + Sized + 'static { #[inline] fn _into_ref(self, cls: PyTypeRef, ctx: &Context) -> PyRef where - Self: std::fmt::Debug, + Self: core::fmt::Debug, { let dict = if cls.slots.flags.has_feature(PyTypeFlags::HAS_DICT) { Some(ctx.new_dict()) @@ -77,7 +77,7 @@ pub trait PyPayload: MaybeTraverse + PyThreadingConstraint + Sized + 'static { #[inline] fn into_exact_ref(self, ctx: &Context) -> PyRefExact where - Self: std::fmt::Debug, + Self: core::fmt::Debug, { unsafe { // Self::into_ref() always returns exact typed PyRef @@ -88,7 +88,7 @@ pub trait PyPayload: MaybeTraverse + PyThreadingConstraint + Sized + 'static { #[inline] fn into_ref(self, ctx: &Context) -> PyRef where - Self: std::fmt::Debug, + Self: core::fmt::Debug, { let cls = Self::class(ctx); self._into_ref(cls.to_owned(), ctx) @@ -97,7 +97,7 @@ pub trait PyPayload: MaybeTraverse + PyThreadingConstraint + Sized + 'static { #[inline] fn into_ref_with_type(self, vm: &VirtualMachine, cls: PyTypeRef) -> PyResult> where - Self: std::fmt::Debug, + Self: core::fmt::Debug, { let exact_class = Self::class(&vm.ctx); if cls.fast_issubclass(exact_class) { @@ -138,11 +138,11 @@ pub trait PyPayload: MaybeTraverse + PyThreadingConstraint + Sized + 'static { } pub trait PyObjectPayload: - PyPayload + std::any::Any + std::fmt::Debug + MaybeTraverse + PyThreadingConstraint + 'static + PyPayload + core::any::Any + core::fmt::Debug + MaybeTraverse + PyThreadingConstraint + 'static { } -impl PyObjectPayload for T {} +impl PyObjectPayload for T {} pub trait SlotOffset { fn offset() -> usize; diff --git a/crates/vm/src/object/traverse.rs b/crates/vm/src/object/traverse.rs index 31bee8becea..2ce0db41a5e 100644 --- a/crates/vm/src/object/traverse.rs +++ b/crates/vm/src/object/traverse.rs @@ -1,4 +1,4 @@ -use std::ptr::NonNull; +use core::ptr::NonNull; use rustpython_common::lock::{PyMutex, PyRwLock}; diff --git a/crates/vm/src/object/traverse_object.rs b/crates/vm/src/object/traverse_object.rs index 281b0e56eb5..075ce5b9513 100644 --- a/crates/vm/src/object/traverse_object.rs +++ b/crates/vm/src/object/traverse_object.rs @@ -1,4 +1,4 @@ -use std::fmt; +use alloc::fmt; use crate::{ PyObject, diff --git a/crates/vm/src/ospath.rs b/crates/vm/src/ospath.rs index 77abbee2cd5..25fcafb74c5 100644 --- a/crates/vm/src/ospath.rs +++ b/crates/vm/src/ospath.rs @@ -230,12 +230,12 @@ impl OsPath { self.path.into_encoded_bytes() } - pub fn to_string_lossy(&self) -> std::borrow::Cow<'_, str> { + pub fn to_string_lossy(&self) -> alloc::borrow::Cow<'_, str> { self.path.to_string_lossy() } - pub fn into_cstring(self, vm: &VirtualMachine) -> PyResult { - std::ffi::CString::new(self.into_bytes()).map_err(|err| err.to_pyexception(vm)) + pub fn into_cstring(self, vm: &VirtualMachine) -> PyResult { + alloc::ffi::CString::new(self.into_bytes()).map_err(|err| err.to_pyexception(vm)) } #[cfg(windows)] diff --git a/crates/vm/src/protocol/buffer.rs b/crates/vm/src/protocol/buffer.rs index 88524a9a9ee..0fe4d15458b 100644 --- a/crates/vm/src/protocol/buffer.rs +++ b/crates/vm/src/protocol/buffer.rs @@ -10,8 +10,9 @@ use crate::{ object::PyObjectPayload, sliceable::SequenceIndexOp, }; +use alloc::borrow::Cow; +use core::{fmt::Debug, ops::Range}; use itertools::Itertools; -use std::{borrow::Cow, fmt::Debug, ops::Range}; pub struct BufferMethods { pub obj_bytes: fn(&PyBuffer) -> BorrowedValue<'_, [u8]>, @@ -21,7 +22,7 @@ pub struct BufferMethods { } impl Debug for BufferMethods { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result { f.debug_struct("BufferMethods") .field("obj_bytes", &(self.obj_bytes as usize)) .field("obj_bytes_mut", &(self.obj_bytes_mut as usize)) @@ -134,8 +135,8 @@ impl PyBuffer { pub(crate) unsafe fn drop_without_release(&mut self) { // SAFETY: requirements forwarded from caller unsafe { - std::ptr::drop_in_place(&mut self.obj); - std::ptr::drop_in_place(&mut self.desc); + core::ptr::drop_in_place(&mut self.obj); + core::ptr::drop_in_place(&mut self.desc); } } } @@ -414,7 +415,7 @@ pub struct VecBuffer { #[pyclass(flags(BASETYPE, DISALLOW_INSTANTIATION))] impl VecBuffer { pub fn take(&self) -> Vec { - std::mem::take(&mut self.data.lock()) + core::mem::take(&mut self.data.lock()) } } diff --git a/crates/vm/src/protocol/callable.rs b/crates/vm/src/protocol/callable.rs index 5280e04e928..fa5e48d58ba 100644 --- a/crates/vm/src/protocol/callable.rs +++ b/crates/vm/src/protocol/callable.rs @@ -61,8 +61,8 @@ enum TraceEvent { Return, } -impl std::fmt::Display for TraceEvent { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { +impl core::fmt::Display for TraceEvent { + fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result { use TraceEvent::*; match self { Call => write!(f, "call"), diff --git a/crates/vm/src/protocol/iter.rs b/crates/vm/src/protocol/iter.rs index f6146543de9..aa6ab6769cd 100644 --- a/crates/vm/src/protocol/iter.rs +++ b/crates/vm/src/protocol/iter.rs @@ -4,8 +4,8 @@ use crate::{ convert::{ToPyObject, ToPyResult}, object::{Traverse, TraverseFn}, }; -use std::borrow::Borrow; -use std::ops::Deref; +use core::borrow::Borrow; +use core::ops::Deref; /// Iterator Protocol // https://docs.python.org/3/c-api/iter.html @@ -223,7 +223,7 @@ where vm: &'a VirtualMachine, obj: O, // creating PyIter is zero-cost length_hint: Option, - _phantom: std::marker::PhantomData, + _phantom: core::marker::PhantomData, } unsafe impl Traverse for PyIterIter<'_, T, O> @@ -244,7 +244,7 @@ where vm, obj, length_hint, - _phantom: std::marker::PhantomData, + _phantom: core::marker::PhantomData, } } } diff --git a/crates/vm/src/protocol/mapping.rs b/crates/vm/src/protocol/mapping.rs index 43dafeb9238..6c200043e35 100644 --- a/crates/vm/src/protocol/mapping.rs +++ b/crates/vm/src/protocol/mapping.rs @@ -22,8 +22,8 @@ pub struct PyMappingSlots { >, } -impl std::fmt::Debug for PyMappingSlots { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { +impl core::fmt::Debug for PyMappingSlots { + fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result { f.write_str("PyMappingSlots") } } @@ -56,8 +56,8 @@ pub struct PyMappingMethods { Option, &PyObject, Option, &VirtualMachine) -> PyResult<()>>, } -impl std::fmt::Debug for PyMappingMethods { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { +impl core::fmt::Debug for PyMappingMethods { + fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result { f.write_str("PyMappingMethods") } } diff --git a/crates/vm/src/protocol/number.rs b/crates/vm/src/protocol/number.rs index c208bf26de8..58891d1d710 100644 --- a/crates/vm/src/protocol/number.rs +++ b/crates/vm/src/protocol/number.rs @@ -1,4 +1,4 @@ -use std::ops::Deref; +use core::ops::Deref; use crossbeam_utils::atomic::AtomicCell; diff --git a/crates/vm/src/protocol/sequence.rs b/crates/vm/src/protocol/sequence.rs index 888ef91565f..cee46a29089 100644 --- a/crates/vm/src/protocol/sequence.rs +++ b/crates/vm/src/protocol/sequence.rs @@ -29,8 +29,8 @@ pub struct PySequenceSlots { pub inplace_repeat: AtomicCell, isize, &VirtualMachine) -> PyResult>>, } -impl std::fmt::Debug for PySequenceSlots { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { +impl core::fmt::Debug for PySequenceSlots { + fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result { f.write_str("PySequenceSlots") } } @@ -83,8 +83,8 @@ pub struct PySequenceMethods { pub inplace_repeat: Option, isize, &VirtualMachine) -> PyResult>, } -impl std::fmt::Debug for PySequenceMethods { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { +impl core::fmt::Debug for PySequenceMethods { + fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result { f.write_str("PySequenceMethods") } } diff --git a/crates/vm/src/py_io.rs b/crates/vm/src/py_io.rs index a8063673a70..5649463b30e 100644 --- a/crates/vm/src/py_io.rs +++ b/crates/vm/src/py_io.rs @@ -3,7 +3,9 @@ use crate::{ builtins::{PyBaseExceptionRef, PyBytes, PyStr}, common::ascii, }; -use std::{fmt, io, ops}; +use alloc::fmt; +use core::ops; +use std::io; pub trait Write { type Error; diff --git a/crates/vm/src/py_serde.rs b/crates/vm/src/py_serde.rs index f9a5f4bc060..945068113f1 100644 --- a/crates/vm/src/py_serde.rs +++ b/crates/vm/src/py_serde.rs @@ -130,7 +130,7 @@ impl<'de> DeserializeSeed<'de> for PyObjectDeserializer<'de> { impl<'de> Visitor<'de> for PyObjectDeserializer<'de> { type Value = PyObjectRef; - fn expecting(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + fn expecting(&self, formatter: &mut core::fmt::Formatter<'_>) -> core::fmt::Result { formatter.write_str("a type that can deserialize in Python") } diff --git a/crates/vm/src/readline.rs b/crates/vm/src/readline.rs index 77402dc6839..d62d520ecbd 100644 --- a/crates/vm/src/readline.rs +++ b/crates/vm/src/readline.rs @@ -5,7 +5,7 @@ use std::{io, path::Path}; -type OtherError = Box; +type OtherError = Box; type OtherResult = Result; pub enum ReadlineResult { diff --git a/crates/vm/src/scope.rs b/crates/vm/src/scope.rs index 4f80e9999ec..74392dc9b73 100644 --- a/crates/vm/src/scope.rs +++ b/crates/vm/src/scope.rs @@ -1,5 +1,5 @@ use crate::{VirtualMachine, builtins::PyDictRef, function::ArgMapping}; -use std::fmt; +use alloc::fmt; #[derive(Clone)] pub struct Scope { diff --git a/crates/vm/src/sequence.rs b/crates/vm/src/sequence.rs index 6e03ad1697e..0bc12fd2631 100644 --- a/crates/vm/src/sequence.rs +++ b/crates/vm/src/sequence.rs @@ -6,8 +6,8 @@ use crate::{ types::PyComparisonOp, vm::{MAX_MEMORY_SIZE, VirtualMachine}, }; +use core::ops::{Deref, Range}; use optional::Optioned; -use std::ops::{Deref, Range}; pub trait MutObjectSequenceOp { type Inner: ?Sized; @@ -100,7 +100,7 @@ where fn mul(&self, vm: &VirtualMachine, n: isize) -> PyResult> { let n = vm.check_repeat_or_overflow_error(self.as_ref().len(), n)?; - if n > 1 && std::mem::size_of_val(self.as_ref()) >= MAX_MEMORY_SIZE / n { + if n > 1 && core::mem::size_of_val(self.as_ref()) >= MAX_MEMORY_SIZE / n { return Err(vm.new_memory_error("")); } diff --git a/crates/vm/src/signal.rs b/crates/vm/src/signal.rs index 1074c8e8f11..4a1b84a1521 100644 --- a/crates/vm/src/signal.rs +++ b/crates/vm/src/signal.rs @@ -1,12 +1,8 @@ #![cfg_attr(target_os = "wasi", allow(dead_code))] use crate::{PyResult, VirtualMachine}; -use std::{ - fmt, - sync::{ - atomic::{AtomicBool, Ordering}, - mpsc, - }, -}; +use alloc::fmt; +use core::sync::atomic::{AtomicBool, Ordering}; +use std::sync::mpsc; pub(crate) const NSIG: usize = 64; static ANY_TRIGGERED: AtomicBool = AtomicBool::new(false); diff --git a/crates/vm/src/sliceable.rs b/crates/vm/src/sliceable.rs index 786b66fb36a..e416f5a1b49 100644 --- a/crates/vm/src/sliceable.rs +++ b/crates/vm/src/sliceable.rs @@ -3,9 +3,9 @@ use crate::{ PyObject, PyResult, VirtualMachine, builtins::{int::PyInt, slice::PySlice}, }; +use core::ops::Range; use malachite_bigint::BigInt; use num_traits::{Signed, ToPrimitive}; -use std::ops::Range; pub trait SliceableSequenceMutOp where diff --git a/crates/vm/src/stdlib/ast/elif_else_clause.rs b/crates/vm/src/stdlib/ast/elif_else_clause.rs index 581fc499b8a..e2a8789dd08 100644 --- a/crates/vm/src/stdlib/ast/elif_else_clause.rs +++ b/crates/vm/src/stdlib/ast/elif_else_clause.rs @@ -3,7 +3,7 @@ use rustpython_compiler_core::SourceFile; pub(super) fn ast_to_object( clause: ruff::ElifElseClause, - mut rest: std::vec::IntoIter, + mut rest: alloc::vec::IntoIter, vm: &VirtualMachine, source_file: &SourceFile, ) -> PyObjectRef { diff --git a/crates/vm/src/stdlib/ast/parameter.rs b/crates/vm/src/stdlib/ast/parameter.rs index 87fa736687b..44fcbb2b464 100644 --- a/crates/vm/src/stdlib/ast/parameter.rs +++ b/crates/vm/src/stdlib/ast/parameter.rs @@ -403,7 +403,7 @@ fn merge_keyword_parameter_defaults( kw_only_args: KeywordParameters, defaults: ParameterDefaults, ) -> Vec { - std::iter::zip(kw_only_args.keywords, defaults.defaults) + core::iter::zip(kw_only_args.keywords, defaults.defaults) .map(|(parameter, default)| ruff::ParameterWithDefault { node_index: Default::default(), parameter, diff --git a/crates/vm/src/stdlib/ast/string.rs b/crates/vm/src/stdlib/ast/string.rs index f3df8d99262..ffa5a3a958a 100644 --- a/crates/vm/src/stdlib/ast/string.rs +++ b/crates/vm/src/stdlib/ast/string.rs @@ -12,7 +12,7 @@ fn ruff_fstring_value_into_iter( }); (0..fstring_value.as_slice().len()).map(move |i| { let tmp = fstring_value.iter_mut().nth(i).unwrap(); - std::mem::replace(tmp, default.clone()) + core::mem::replace(tmp, default.clone()) }) } @@ -28,7 +28,7 @@ fn ruff_fstring_element_into_iter( (0..fstring_element.into_iter().len()).map(move |i| { let fstring_element = &mut fstring_element; let tmp = fstring_element.into_iter().nth(i).unwrap(); - std::mem::replace(tmp, default.clone()) + core::mem::replace(tmp, default.clone()) }) } diff --git a/crates/vm/src/stdlib/atexit.rs b/crates/vm/src/stdlib/atexit.rs index b1832b5481d..2286c36f1db 100644 --- a/crates/vm/src/stdlib/atexit.rs +++ b/crates/vm/src/stdlib/atexit.rs @@ -34,7 +34,7 @@ mod atexit { #[pyfunction] pub fn _run_exitfuncs(vm: &VirtualMachine) { - let funcs: Vec<_> = std::mem::take(&mut *vm.state.atexit_funcs.lock()); + let funcs: Vec<_> = core::mem::take(&mut *vm.state.atexit_funcs.lock()); for (func, args) in funcs.into_iter().rev() { if let Err(e) = func.call(args, vm) { let exit = e.fast_isinstance(vm.ctx.exceptions.system_exit); diff --git a/crates/vm/src/stdlib/builtins.rs b/crates/vm/src/stdlib/builtins.rs index 7cd91f8b4b7..c82161fc553 100644 --- a/crates/vm/src/stdlib/builtins.rs +++ b/crates/vm/src/stdlib/builtins.rs @@ -175,7 +175,7 @@ mod builtins { let source = source.borrow_bytes(); // TODO: compiler::compile should probably get bytes - let source = std::str::from_utf8(&source) + let source = core::str::from_utf8(&source) .map_err(|e| vm.new_unicode_decode_error(e.to_string()))?; let flags = args.flags.map_or(Ok(0), |v| v.try_to_primitive(vm))?; @@ -333,7 +333,7 @@ mod builtins { )); } - let source = std::str::from_utf8(source).map_err(|err| { + let source = core::str::from_utf8(source).map_err(|err| { let msg = format!( "(unicode error) 'utf-8' codec can't decode byte 0x{:x?} in position {}: invalid start byte", source[err.valid_up_to()], @@ -605,7 +605,7 @@ mod builtins { } let candidates = match args.args.len().cmp(&1) { - std::cmp::Ordering::Greater => { + core::cmp::Ordering::Greater => { if default.is_some() { return Err(vm.new_type_error(format!( "Cannot specify a default for {func_name}() with multiple positional arguments" @@ -613,8 +613,8 @@ mod builtins { } args.args } - std::cmp::Ordering::Equal => args.args[0].try_to_value(vm)?, - std::cmp::Ordering::Less => { + core::cmp::Ordering::Equal => args.args[0].try_to_value(vm)?, + core::cmp::Ordering::Less => { // zero arguments means type error: return Err( vm.new_type_error(format!("{func_name} expected at least 1 argument, got 0")) diff --git a/crates/vm/src/stdlib/codecs.rs b/crates/vm/src/stdlib/codecs.rs index 821b313090c..1661eef1750 100644 --- a/crates/vm/src/stdlib/codecs.rs +++ b/crates/vm/src/stdlib/codecs.rs @@ -270,7 +270,7 @@ mod _codecs { wide.len() as i32, std::ptr::null_mut(), 0, - std::ptr::null(), + core::ptr::null(), std::ptr::null_mut(), ) }; @@ -291,7 +291,7 @@ mod _codecs { wide.len() as i32, buffer.as_mut_ptr().cast(), size, - std::ptr::null(), + core::ptr::null(), if errors == "strict" { &mut used_default_char } else { @@ -472,7 +472,7 @@ mod _codecs { wide.len() as i32, std::ptr::null_mut(), 0, - std::ptr::null(), + core::ptr::null(), std::ptr::null_mut(), ) }; @@ -493,7 +493,7 @@ mod _codecs { wide.len() as i32, buffer.as_mut_ptr().cast(), size, - std::ptr::null(), + core::ptr::null(), if errors == "strict" { &mut used_default_char } else { diff --git a/crates/vm/src/stdlib/collections.rs b/crates/vm/src/stdlib/collections.rs index eae56968cba..1249fa9315d 100644 --- a/crates/vm/src/stdlib/collections.rs +++ b/crates/vm/src/stdlib/collections.rs @@ -22,9 +22,9 @@ mod _collections { }, utils::collection_repr, }; + use alloc::collections::VecDeque; + use core::cmp::max; use crossbeam_utils::atomic::AtomicCell; - use std::cmp::max; - use std::collections::VecDeque; #[pyattr] #[pyclass(module = "collections", name = "deque", unhashable = true)] @@ -157,7 +157,7 @@ mod _collections { let mut created = VecDeque::from(elements); let mut borrowed = self.borrow_deque_mut(); created.append(&mut borrowed); - std::mem::swap(&mut created, &mut borrowed); + core::mem::swap(&mut created, &mut borrowed); Ok(()) } @@ -426,7 +426,7 @@ mod _collections { inner.get(index).map(|r| r.as_ref()) } - fn do_lock(&self) -> impl std::ops::Deref { + fn do_lock(&self) -> impl core::ops::Deref { self.borrow_deque() } } @@ -484,7 +484,7 @@ mod _collections { // `maxlen` is better to be defined as UnsafeCell in common practice, // but then more type works without any safety benefits let unsafe_maxlen = - &zelf.maxlen as *const _ as *const std::cell::UnsafeCell>; + &zelf.maxlen as *const _ as *const core::cell::UnsafeCell>; *(*unsafe_maxlen).get() = maxlen; } if let Some(elements) = elements { diff --git a/crates/vm/src/stdlib/ctypes.rs b/crates/vm/src/stdlib/ctypes.rs index a9c0636bd12..f3b6dd25aca 100644 --- a/crates/vm/src/stdlib/ctypes.rs +++ b/crates/vm/src/stdlib/ctypes.rs @@ -15,11 +15,11 @@ use crate::{ class::PyClassImpl, types::TypeDataRef, }; -use std::ffi::{ +use core::ffi::{ c_double, c_float, c_int, c_long, c_longlong, c_schar, c_short, c_uchar, c_uint, c_ulong, c_ulonglong, c_ushort, }; -use std::mem; +use core::mem; use widestring::WideChar; pub use array::PyCArray; @@ -387,7 +387,7 @@ pub(crate) mod _ctypes { const RTLD_GLOBAL: i32 = 0; #[pyattr] - const SIZEOF_TIME_T: usize = std::mem::size_of::(); + const SIZEOF_TIME_T: usize = core::mem::size_of::(); #[pyattr] const CTYPES_MAX_ARGCOUNT: usize = 1024; @@ -535,11 +535,11 @@ pub(crate) mod _ctypes { { return Ok(super::get_size(type_str.as_ref())); } - return Ok(std::mem::size_of::()); + return Ok(core::mem::size_of::()); } // Pointer types if type_obj.fast_issubclass(PyCPointer::static_type()) { - return Ok(std::mem::size_of::()); + return Ok(core::mem::size_of::()); } return Err(vm.new_type_error("this type has no size")); } @@ -550,7 +550,7 @@ pub(crate) mod _ctypes { return Ok(cdata.size()); } if obj.fast_isinstance(PyCPointer::static_type()) { - return Ok(std::mem::size_of::()); + return Ok(core::mem::size_of::()); } Err(vm.new_type_error("this type has no size")) @@ -596,13 +596,17 @@ pub(crate) mod _ctypes { } None => { // dlopen(NULL, mode) to get the current process handle (for pythonapi) - let handle = unsafe { libc::dlopen(std::ptr::null(), mode) }; + let handle = unsafe { libc::dlopen(core::ptr::null(), mode) }; if handle.is_null() { let err = unsafe { libc::dlerror() }; let msg = if err.is_null() { "dlopen() error".to_string() } else { - unsafe { std::ffi::CStr::from_ptr(err).to_string_lossy().into_owned() } + unsafe { + core::ffi::CStr::from_ptr(err) + .to_string_lossy() + .into_owned() + } }; return Err(vm.new_os_error(msg)); } @@ -641,7 +645,7 @@ pub(crate) mod _ctypes { name: crate::builtins::PyStrRef, vm: &VirtualMachine, ) -> PyResult { - let symbol_name = std::ffi::CString::new(name.as_str()) + let symbol_name = alloc::ffi::CString::new(name.as_str()) .map_err(|_| vm.new_value_error("symbol name contains null byte"))?; // Clear previous error @@ -652,7 +656,11 @@ pub(crate) mod _ctypes { // Check for error via dlerror first let err = unsafe { libc::dlerror() }; if !err.is_null() { - let msg = unsafe { std::ffi::CStr::from_ptr(err).to_string_lossy().into_owned() }; + let msg = unsafe { + core::ffi::CStr::from_ptr(err) + .to_string_lossy() + .into_owned() + }; return Err(vm.new_os_error(msg)); } @@ -851,7 +859,7 @@ pub(crate) mod _ctypes { } if obj.fast_isinstance(PyCPointer::static_type()) { // Pointer alignment is always pointer size - return Ok(std::mem::align_of::()); + return Ok(core::mem::align_of::()); } if obj.fast_isinstance(PyCUnion::static_type()) { // Calculate alignment from _fields_ @@ -914,7 +922,7 @@ pub(crate) mod _ctypes { #[pyfunction] fn resize(obj: PyObjectRef, size: isize, vm: &VirtualMachine) -> PyResult<()> { - use std::borrow::Cow; + use alloc::borrow::Cow; // 1. Get StgInfo from object's class (validates ctypes instance) let stg_info = obj @@ -1148,8 +1156,8 @@ pub(crate) mod _ctypes { } let raw_ptr = ptr as *mut crate::object::PyObject; unsafe { - let obj = crate::PyObjectRef::from_raw(std::ptr::NonNull::new_unchecked(raw_ptr)); - let obj = std::mem::ManuallyDrop::new(obj); + let obj = crate::PyObjectRef::from_raw(core::ptr::NonNull::new_unchecked(raw_ptr)); + let obj = core::mem::ManuallyDrop::new(obj); Ok((*obj).clone()) } } @@ -1208,12 +1216,12 @@ pub(crate) mod _ctypes { FORMAT_MESSAGE_ALLOCATE_BUFFER | FORMAT_MESSAGE_FROM_SYSTEM | FORMAT_MESSAGE_IGNORE_INSERTS, - std::ptr::null(), + core::ptr::null(), error_code, 0, &mut buffer as *mut *mut u16 as *mut u16, 0, - std::ptr::null(), + core::ptr::null(), ) }; @@ -1280,7 +1288,7 @@ pub(crate) mod _ctypes { let vtable = *iunknown; debug_assert!(!vtable.is_null(), "IUnknown vtable is null"); let addref_fn: extern "system" fn(*mut std::ffi::c_void) -> u32 = - std::mem::transmute(*vtable.add(1)); // AddRef is index 1 + core::mem::transmute(*vtable.add(1)); // AddRef is index 1 addref_fn(src_ptr as *mut std::ffi::c_void); } } diff --git a/crates/vm/src/stdlib/ctypes/array.rs b/crates/vm/src/stdlib/ctypes/array.rs index 25708b57f8e..168da2bcc01 100644 --- a/crates/vm/src/stdlib/ctypes/array.rs +++ b/crates/vm/src/stdlib/ctypes/array.rs @@ -631,7 +631,7 @@ impl PyCArray { let ptr_val = usize::from_ne_bytes( ptr_bytes .try_into() - .unwrap_or([0; std::mem::size_of::()]), + .unwrap_or([0; core::mem::size_of::()]), ); if ptr_val == 0 { return Ok(vm.ctx.none()); @@ -643,7 +643,7 @@ impl PyCArray { while *ptr.add(len) != 0 { len += 1; } - let bytes = std::slice::from_raw_parts(ptr, len); + let bytes = core::slice::from_raw_parts(ptr, len); Ok(vm.ctx.new_bytes(bytes.to_vec()).into()) } } @@ -656,7 +656,7 @@ impl PyCArray { let ptr_val = usize::from_ne_bytes( ptr_bytes .try_into() - .unwrap_or([0; std::mem::size_of::()]), + .unwrap_or([0; core::mem::size_of::()]), ); if ptr_val == 0 { return Ok(vm.ctx.none()); @@ -668,10 +668,10 @@ impl PyCArray { let mut pos = 0usize; loop { let code = if WCHAR_SIZE == 2 { - let bytes = std::slice::from_raw_parts(ptr.add(pos), 2); + let bytes = core::slice::from_raw_parts(ptr.add(pos), 2); u16::from_ne_bytes([bytes[0], bytes[1]]) as u32 } else { - let bytes = std::slice::from_raw_parts(ptr.add(pos), 4); + let bytes = core::slice::from_raw_parts(ptr.add(pos), 4); u32::from_ne_bytes([bytes[0], bytes[1], bytes[2], bytes[3]]) }; if code == 0 { @@ -1136,7 +1136,7 @@ impl AsBuffer for PyCArray { len: buffer_len, readonly: false, itemsize, - format: std::borrow::Cow::Owned(fmt), + format: alloc::borrow::Cow::Owned(fmt), dim_desc, } } else { @@ -1291,7 +1291,7 @@ fn add_wchar_array_getsets(array_type: &Py, vm: &VirtualMachine) { // Linux/macOS: sizeof(wchar_t) == 4 (UTF-32) /// Size of wchar_t on this platform -pub(super) const WCHAR_SIZE: usize = std::mem::size_of::(); +pub(super) const WCHAR_SIZE: usize = core::mem::size_of::(); /// Read a single wchar_t from bytes (platform-endian) #[inline] diff --git a/crates/vm/src/stdlib/ctypes/base.rs b/crates/vm/src/stdlib/ctypes/base.rs index 0f859b3d10b..1f9eaeef56a 100644 --- a/crates/vm/src/stdlib/ctypes/base.rs +++ b/crates/vm/src/stdlib/ctypes/base.rs @@ -7,15 +7,15 @@ use crate::types::{GetDescriptor, Representable}; use crate::{ AsObject, Py, PyObject, PyObjectRef, PyPayload, PyResult, TryFromObject, VirtualMachine, }; +use alloc::borrow::Cow; +use core::ffi::{ + c_double, c_float, c_int, c_long, c_longlong, c_short, c_uint, c_ulong, c_ulonglong, c_ushort, +}; +use core::fmt::Debug; +use core::mem; use crossbeam_utils::atomic::AtomicCell; use num_traits::{Signed, ToPrimitive}; use rustpython_common::lock::PyRwLock; -use std::borrow::Cow; -use std::ffi::{ - c_double, c_float, c_int, c_long, c_longlong, c_short, c_uint, c_ulong, c_ulonglong, c_ushort, -}; -use std::fmt::Debug; -use std::mem; use widestring::WideChar; // StgInfo - Storage information for ctypes types @@ -105,8 +105,8 @@ pub struct StgInfo { unsafe impl Send for StgInfo {} unsafe impl Sync for StgInfo {} -impl std::fmt::Debug for StgInfo { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { +impl core::fmt::Debug for StgInfo { + fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result { f.debug_struct("StgInfo") .field("initialized", &self.initialized) .field("size", &self.size) @@ -227,7 +227,7 @@ impl StgInfo { libffi::middle::Type::structure(self.ffi_field_types.iter().cloned()) } else if self.size <= MAX_FFI_STRUCT_SIZE { // Small struct without field types: use bytes array - libffi::middle::Type::structure(std::iter::repeat_n( + libffi::middle::Type::structure(core::iter::repeat_n( libffi::middle::Type::u8(), self.size, )) @@ -242,9 +242,9 @@ impl StgInfo { libffi::middle::Type::pointer() } else if let Some(ref fmt) = self.format { let elem_type = Self::format_to_ffi_type(fmt); - libffi::middle::Type::structure(std::iter::repeat_n(elem_type, self.length)) + libffi::middle::Type::structure(core::iter::repeat_n(elem_type, self.length)) } else { - libffi::middle::Type::structure(std::iter::repeat_n( + libffi::middle::Type::structure(core::iter::repeat_n( libffi::middle::Type::u8(), self.size, )) @@ -373,10 +373,10 @@ pub(super) static CDATA_BUFFER_METHODS: BufferMethods = BufferMethods { /// Convert Vec to Vec by reinterpreting the memory (same allocation). fn vec_to_bytes(vec: Vec) -> Vec { - let len = vec.len() * std::mem::size_of::(); - let cap = vec.capacity() * std::mem::size_of::(); + let len = vec.len() * core::mem::size_of::(); + let cap = vec.capacity() * core::mem::size_of::(); let ptr = vec.as_ptr() as *mut u8; - std::mem::forget(vec); + core::mem::forget(vec); unsafe { Vec::from_raw_parts(ptr, len, cap) } } @@ -406,7 +406,7 @@ pub(super) fn str_to_wchar_bytes(s: &str, vm: &VirtualMachine) -> (PyObjectRef, let wchars: Vec = s .chars() .map(|c| c as libc::wchar_t) - .chain(std::iter::once(0)) + .chain(core::iter::once(0)) .collect(); let ptr = wchars.as_ptr() as usize; let bytes = vec_to_bytes(wchars); @@ -486,7 +486,7 @@ impl PyCData { pub unsafe fn at_address(ptr: *const u8, size: usize) -> Self { // = PyCData_AtAddress // SAFETY: Caller must ensure ptr is valid for the lifetime of returned PyCData - let slice: &'static [u8] = unsafe { std::slice::from_raw_parts(ptr, size) }; + let slice: &'static [u8] = unsafe { core::slice::from_raw_parts(ptr, size) }; PyCData { buffer: PyRwLock::new(Cow::Borrowed(slice)), base: PyRwLock::new(None), @@ -534,7 +534,7 @@ impl PyCData { ) -> Self { // = PyCData_FromBaseObj // SAFETY: ptr points into base_obj's buffer, kept alive via base reference - let slice: &'static [u8] = unsafe { std::slice::from_raw_parts(ptr, size) }; + let slice: &'static [u8] = unsafe { core::slice::from_raw_parts(ptr, size) }; PyCData { buffer: PyRwLock::new(Cow::Borrowed(slice)), base: PyRwLock::new(Some(base_obj)), @@ -561,7 +561,7 @@ impl PyCData { vm: &VirtualMachine, ) -> Self { // SAFETY: Caller must ensure ptr is valid for the lifetime of source - let slice: &'static [u8] = unsafe { std::slice::from_raw_parts(ptr, size) }; + let slice: &'static [u8] = unsafe { core::slice::from_raw_parts(ptr, size) }; // Python stores the reference in a dict with key "-1" (unique_key pattern) let objects_dict = vm.ctx.new_dict(); @@ -707,7 +707,7 @@ impl PyCData { // (e.g., from from_address pointing to a ctypes buffer) unsafe { let ptr = slice.as_ptr() as *mut u8; - std::ptr::copy_nonoverlapping(bytes.as_ptr(), ptr.add(offset), bytes.len()); + core::ptr::copy_nonoverlapping(bytes.as_ptr(), ptr.add(offset), bytes.len()); } } Cow::Owned(_) => { @@ -893,7 +893,7 @@ impl PyCData { if let Some(bytes_val) = value.downcast_ref::() { let src = bytes_val.as_bytes(); let to_copy = PyCField::bytes_for_char_array(src); - let copy_len = std::cmp::min(to_copy.len(), size); + let copy_len = core::cmp::min(to_copy.len(), size); self.write_bytes_at_offset(offset, &to_copy[..copy_len]); self.keep_ref(index, value, vm)?; return Ok(()); @@ -936,7 +936,7 @@ impl PyCData { array_buffer.as_ptr() as usize }; let addr_bytes = buffer_addr.to_ne_bytes(); - let len = std::cmp::min(addr_bytes.len(), size); + let len = core::cmp::min(addr_bytes.len(), size); self.write_bytes_at_offset(offset, &addr_bytes[..len]); self.keep_ref(index, value, vm)?; return Ok(()); @@ -1364,7 +1364,7 @@ impl PyCField { if let Some(bytes) = value.downcast_ref::() { let src = bytes.as_bytes(); let mut result = vec![0u8; size]; - let len = std::cmp::min(src.len(), size); + let len = core::cmp::min(src.len(), size); result[..len].copy_from_slice(&src[..len]); Ok(result) } @@ -1372,7 +1372,7 @@ impl PyCField { else if let Some(cdata) = value.downcast_ref::() { let buffer = cdata.buffer.read(); let mut result = vec![0u8; size]; - let len = std::cmp::min(buffer.len(), size); + let len = core::cmp::min(buffer.len(), size); result[..len].copy_from_slice(&buffer[..len]); Ok(result) } @@ -1473,7 +1473,7 @@ impl PyCField { let (converted, ptr) = ensure_z_null_terminated(bytes, vm); let mut result = vec![0u8; size]; let addr_bytes = ptr.to_ne_bytes(); - let len = std::cmp::min(addr_bytes.len(), size); + let len = core::cmp::min(addr_bytes.len(), size); result[..len].copy_from_slice(&addr_bytes[..len]); return Ok((result, Some(converted))); } @@ -1482,7 +1482,7 @@ impl PyCField { let v = int_val.as_bigint().to_usize().unwrap_or(0); let mut result = vec![0u8; size]; let bytes = v.to_ne_bytes(); - let len = std::cmp::min(bytes.len(), size); + let len = core::cmp::min(bytes.len(), size); result[..len].copy_from_slice(&bytes[..len]); return Ok((result, None)); } @@ -1498,7 +1498,7 @@ impl PyCField { let (holder, ptr) = str_to_wchar_bytes(s.as_str(), vm); let mut result = vec![0u8; size]; let addr_bytes = ptr.to_ne_bytes(); - let len = std::cmp::min(addr_bytes.len(), size); + let len = core::cmp::min(addr_bytes.len(), size); result[..len].copy_from_slice(&addr_bytes[..len]); return Ok((result, Some(holder))); } @@ -1507,7 +1507,7 @@ impl PyCField { let v = int_val.as_bigint().to_usize().unwrap_or(0); let mut result = vec![0u8; size]; let bytes = v.to_ne_bytes(); - let len = std::cmp::min(bytes.len(), size); + let len = core::cmp::min(bytes.len(), size); result[..len].copy_from_slice(&bytes[..len]); return Ok((result, None)); } @@ -1523,7 +1523,7 @@ impl PyCField { let v = int_val.as_bigint().to_usize().unwrap_or(0); let mut result = vec![0u8; size]; let bytes = v.to_ne_bytes(); - let len = std::cmp::min(bytes.len(), size); + let len = core::cmp::min(bytes.len(), size); result[..len].copy_from_slice(&bytes[..len]); return Ok((result, None)); } @@ -2078,7 +2078,7 @@ pub(super) fn bytes_to_pyobject( if ptr == 0 { return Ok(vm.ctx.none()); } - let c_str = unsafe { std::ffi::CStr::from_ptr(ptr as _) }; + let c_str = unsafe { core::ffi::CStr::from_ptr(ptr as _) }; Ok(vm.ctx.new_bytes(c_str.to_bytes().to_vec()).into()) } "Z" => { @@ -2089,7 +2089,7 @@ pub(super) fn bytes_to_pyobject( } let len = unsafe { libc::wcslen(ptr as *const libc::wchar_t) }; let wchars = - unsafe { std::slice::from_raw_parts(ptr as *const libc::wchar_t, len) }; + unsafe { core::slice::from_raw_parts(ptr as *const libc::wchar_t, len) }; let s: String = wchars .iter() .filter_map(|&c| char::from_u32(c as u32)) @@ -2149,7 +2149,7 @@ pub(super) fn get_usize_attr( /// Read a pointer value from buffer #[inline] pub(super) fn read_ptr_from_buffer(buffer: &[u8]) -> usize { - const PTR_SIZE: usize = std::mem::size_of::(); + const PTR_SIZE: usize = core::mem::size_of::(); if buffer.len() >= PTR_SIZE { usize::from_ne_bytes(buffer[..PTR_SIZE].try_into().unwrap()) } else { @@ -2242,7 +2242,7 @@ pub(super) fn get_field_size(field_type: &PyObject, vm: &VirtualMachine) -> PyRe return Ok(s); } - Ok(std::mem::size_of::()) + Ok(core::mem::size_of::()) } /// Get the alignment of a ctypes field type diff --git a/crates/vm/src/stdlib/ctypes/function.rs b/crates/vm/src/stdlib/ctypes/function.rs index 55a42f0ba15..5906bc91cd4 100644 --- a/crates/vm/src/stdlib/ctypes/function.rs +++ b/crates/vm/src/stdlib/ctypes/function.rs @@ -16,6 +16,9 @@ use crate::{ types::{AsBuffer, Callable, Constructor, Initializer, Representable}, vm::thread::with_current_vm, }; +use alloc::borrow::Cow; +use core::ffi::c_void; +use core::fmt::Debug; use libffi::{ low, middle::{Arg, Cif, Closure, CodePtr, Type}, @@ -23,9 +26,6 @@ use libffi::{ use libloading::Symbol; use num_traits::{Signed, ToPrimitive}; use rustpython_common::lock::PyRwLock; -use std::borrow::Cow; -use std::ffi::c_void; -use std::fmt::Debug; // Internal function addresses for special ctypes functions pub(super) const INTERNAL_CAST_ADDR: usize = 1; @@ -37,7 +37,7 @@ std::thread_local! { /// Thread-local storage for ctypes errno /// This is separate from the system errno - ctypes swaps them during FFI calls /// when use_errno=True is specified. - static CTYPES_LOCAL_ERRNO: std::cell::Cell = const { std::cell::Cell::new(0) }; + static CTYPES_LOCAL_ERRNO: core::cell::Cell = const { core::cell::Cell::new(0) }; } /// Get ctypes thread-local errno value @@ -79,7 +79,7 @@ where #[cfg(windows)] std::thread_local! { /// Thread-local storage for ctypes last_error (Windows only) - static CTYPES_LOCAL_LAST_ERROR: std::cell::Cell = const { std::cell::Cell::new(0) }; + static CTYPES_LOCAL_LAST_ERROR: core::cell::Cell = const { core::cell::Cell::new(0) }; } #[cfg(windows)] @@ -135,14 +135,14 @@ fn ffi_type_from_tag(tag: u8) -> Type { b'i' => Type::i32(), b'I' => Type::u32(), b'l' => { - if std::mem::size_of::() == 8 { + if core::mem::size_of::() == 8 { Type::i64() } else { Type::i32() } } b'L' => { - if std::mem::size_of::() == 8 { + if core::mem::size_of::() == 8 { Type::u64() } else { Type::u32() @@ -154,7 +154,7 @@ fn ffi_type_from_tag(tag: u8) -> Type { b'd' | b'g' => Type::f64(), b'?' => Type::u8(), b'u' => { - if std::mem::size_of::() == 2 { + if core::mem::size_of::() == 2 { Type::u16() } else { Type::u32() @@ -207,7 +207,7 @@ fn convert_to_pointer(value: &PyObject, vm: &VirtualMachine) -> PyResult value from buffer if let Some(simple) = value.downcast_ref::() { let buffer = simple.0.buffer.read(); - if buffer.len() >= std::mem::size_of::() { + if buffer.len() >= core::mem::size_of::() { let addr = super::base::read_ptr_from_buffer(&buffer); return Ok(FfiArgValue::Pointer(addr)); } @@ -283,7 +283,7 @@ fn conv_param(value: &PyObject, vm: &VirtualMachine) -> PyResult { let wide: Vec = s .as_str() .encode_utf16() - .chain(std::iter::once(0)) + .chain(core::iter::once(0)) .collect(); let wide_bytes: Vec = wide.iter().flat_map(|&x| x.to_ne_bytes()).collect(); let keep = vm.ctx.new_bytes(wide_bytes); @@ -499,7 +499,7 @@ impl Initializer for PyCFuncPtrType { new_type.check_not_initialized(vm)?; - let ptr_size = std::mem::size_of::(); + let ptr_size = core::mem::size_of::(); let mut stg_info = StgInfo::new(ptr_size, ptr_size); stg_info.format = Some("X{}".to_string()); stg_info.length = 1; @@ -552,7 +552,7 @@ pub(super) struct PyCFuncPtr { } impl Debug for PyCFuncPtr { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result { f.debug_struct("PyCFuncPtr") .field("func_ptr", &self.get_func_ptr()) .finish() @@ -567,9 +567,9 @@ fn extract_ptr_from_arg(arg: &PyObject, vm: &VirtualMachine) -> PyResult } if let Some(simple) = arg.downcast_ref::() { let buffer = simple.0.buffer.read(); - if buffer.len() >= std::mem::size_of::() { + if buffer.len() >= core::mem::size_of::() { return Ok(usize::from_ne_bytes( - buffer[..std::mem::size_of::()].try_into().unwrap(), + buffer[..core::mem::size_of::()].try_into().unwrap(), )); } } @@ -612,7 +612,7 @@ fn string_at_impl(ptr: usize, size: isize, vm: &VirtualMachine) -> PyResult { } size_usize }; - let bytes = unsafe { std::slice::from_raw_parts(ptr, len) }; + let bytes = unsafe { core::slice::from_raw_parts(ptr, len) }; Ok(vm.ctx.new_bytes(bytes.to_vec()).into()) } @@ -627,12 +627,12 @@ fn wstring_at_impl(ptr: usize, size: isize, vm: &VirtualMachine) -> PyResult { } else { // Overflow check for huge size values let size_usize = size as usize; - if size_usize > isize::MAX as usize / std::mem::size_of::() { + if size_usize > isize::MAX as usize / core::mem::size_of::() { return Err(vm.new_overflow_error("string too long")); } size_usize }; - let wchars = unsafe { std::slice::from_raw_parts(w_ptr, len) }; + let wchars = unsafe { core::slice::from_raw_parts(w_ptr, len) }; // Windows: wchar_t = u16 (UTF-16) -> use Wtf8Buf::from_wide // macOS/Linux: wchar_t = i32 (UTF-32) -> convert via char::from_u32 @@ -815,7 +815,7 @@ impl Constructor for PyCFuncPtr { // 3. Tuple argument: (name, dll) form // 4. Callable: callback creation - let ptr_size = std::mem::size_of::(); + let ptr_size = core::mem::size_of::(); if args.args.is_empty() { return PyCFuncPtr { @@ -1513,11 +1513,11 @@ fn convert_raw_result( RawResult::Void => return None, RawResult::Pointer(ptr) => { let bytes = ptr.to_ne_bytes(); - (bytes.to_vec(), std::mem::size_of::()) + (bytes.to_vec(), core::mem::size_of::()) } RawResult::Value(val) => { let bytes = val.to_ne_bytes(); - (bytes.to_vec(), std::mem::size_of::()) + (bytes.to_vec(), core::mem::size_of::()) } }; @@ -1702,7 +1702,7 @@ impl Callable for PyCFuncPtr { None => { debug_assert!(false, "NULL function pointer"); // In release mode, this will crash - CodePtr(std::ptr::null_mut()) + CodePtr(core::ptr::null_mut()) } }; @@ -1758,7 +1758,7 @@ impl AsBuffer for PyCFuncPtr { stg_info.size, ) } else { - (Cow::Borrowed("X{}"), std::mem::size_of::()) + (Cow::Borrowed("X{}"), core::mem::size_of::()) }; let desc = BufferDescriptor { len: itemsize, @@ -1902,7 +1902,7 @@ fn ffi_to_python(ty: &Py, ptr: *const c_void, vm: &VirtualMachine) -> Py if cstr_ptr.is_null() { vm.ctx.none() } else { - let cstr = std::ffi::CStr::from_ptr(cstr_ptr); + let cstr = core::ffi::CStr::from_ptr(cstr_ptr); vm.ctx.new_bytes(cstr.to_bytes().to_vec()).into() } } @@ -1916,7 +1916,7 @@ fn ffi_to_python(ty: &Py, ptr: *const c_void, vm: &VirtualMachine) -> Py while *wstr_ptr.add(len) != 0 { len += 1; } - let slice = std::slice::from_raw_parts(wstr_ptr, len); + let slice = core::slice::from_raw_parts(wstr_ptr, len); // Windows: wchar_t = u16 (UTF-16) -> use Wtf8Buf::from_wide // Unix: wchar_t = i32 (UTF-32) -> convert via char::from_u32 #[cfg(windows)] @@ -2113,7 +2113,7 @@ pub(super) struct PyCThunk { } impl Debug for PyCThunk { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result { f.debug_struct("PyCThunk") .field("callable", &self.callable) .finish() diff --git a/crates/vm/src/stdlib/ctypes/library.rs b/crates/vm/src/stdlib/ctypes/library.rs index 7512ce29d8a..35ccb433845 100644 --- a/crates/vm/src/stdlib/ctypes/library.rs +++ b/crates/vm/src/stdlib/ctypes/library.rs @@ -1,9 +1,9 @@ use crate::VirtualMachine; +use alloc::fmt; use libloading::Library; use rustpython_common::lock::{PyMutex, PyRwLock}; use std::collections::HashMap; use std::ffi::OsStr; -use std::fmt; #[cfg(unix)] use libloading::os::unix::Library as UnixLibrary; @@ -54,7 +54,7 @@ impl SharedLibrary { // On Windows: HMODULE (*mut c_void) // On Unix: *mut c_void from dlopen // We use transmute_copy to read the handle without consuming the Library - unsafe { std::mem::transmute_copy::(l) } + unsafe { core::mem::transmute_copy::(l) } } else { 0 } diff --git a/crates/vm/src/stdlib/ctypes/pointer.rs b/crates/vm/src/stdlib/ctypes/pointer.rs index ae97b741b3c..f564fd1965c 100644 --- a/crates/vm/src/stdlib/ctypes/pointer.rs +++ b/crates/vm/src/stdlib/ctypes/pointer.rs @@ -8,8 +8,8 @@ use crate::{ class::StaticType, function::{FuncArgs, OptionalArg}, }; +use alloc::borrow::Cow; use num_traits::ToPrimitive; -use std::borrow::Cow; #[pyclass(name = "PyCPointerType", base = PyType, module = "_ctypes")] #[derive(Debug)] @@ -37,7 +37,7 @@ impl Initializer for PyCPointerType { .and_then(|obj| obj.downcast::().ok()); // Initialize StgInfo for pointer type - let pointer_size = std::mem::size_of::(); + let pointer_size = core::mem::size_of::(); let mut stg_info = StgInfo::new(pointer_size, pointer_size); stg_info.proto = proto; stg_info.paramfunc = super::base::ParamFunc::Pointer; @@ -232,7 +232,7 @@ impl Constructor for PyCPointer { // Create a new PyCPointer instance with NULL pointer (all zeros) // Initial contents is set via __init__ if provided - let cdata = PyCData::from_bytes(vec![0u8; std::mem::size_of::()], None); + let cdata = PyCData::from_bytes(vec![0u8; core::mem::size_of::()], None); // pointer instance has b_length set to 2 (for index 0 and 1) cdata.length.store(2); PyCPointer(cdata) @@ -299,7 +299,7 @@ impl PyCPointer { let proto_type = stg_info.proto(); let element_size = proto_type .stg_info_opt() - .map_or(std::mem::size_of::(), |info| info.size); + .map_or(core::mem::size_of::(), |info| info.size); // Create instance that references the memory directly // PyCData.into_ref_with_type works for all ctypes (simple, structure, union, array, pointer) @@ -383,7 +383,7 @@ impl PyCPointer { let proto_type = stg_info.proto(); let element_size = proto_type .stg_info_opt() - .map_or(std::mem::size_of::(), |info| info.size); + .map_or(core::mem::size_of::(), |info| info.size); // offset = index * iteminfo->size let offset = index * element_size as isize; @@ -468,7 +468,7 @@ impl PyCPointer { let element_size = if let Some(ref proto_type) = stg_info.proto { proto_type.stg_info_opt().expect("proto has StgInfo").size } else { - std::mem::size_of::() + core::mem::size_of::() }; let type_code = stg_info .proto @@ -489,7 +489,7 @@ impl PyCPointer { // Optimized contiguous copy let start_addr = (ptr_value as isize + start * element_size as isize) as *const u8; unsafe { - result.extend_from_slice(std::slice::from_raw_parts(start_addr, len)); + result.extend_from_slice(core::slice::from_raw_parts(start_addr, len)); } } else { let mut cur = start; @@ -510,7 +510,7 @@ impl PyCPointer { return Ok(vm.ctx.new_str("").into()); } let mut result = String::with_capacity(len); - let wchar_size = std::mem::size_of::(); + let wchar_size = core::mem::size_of::(); let mut cur = start; for _ in 0..len { let addr = (ptr_value as isize + cur * wchar_size as isize) as *const libc::wchar_t; @@ -578,7 +578,7 @@ impl PyCPointer { let element_size = proto_type .stg_info_opt() - .map_or(std::mem::size_of::(), |info| info.size); + .map_or(core::mem::size_of::(), |info| info.size); // Calculate address let offset = index * element_size as isize; @@ -595,7 +595,7 @@ impl PyCPointer { let copy_len = src_buffer.len().min(element_size); unsafe { let dest_ptr = addr as *mut u8; - std::ptr::copy_nonoverlapping(src_buffer.as_ptr(), dest_ptr, copy_len); + core::ptr::copy_nonoverlapping(src_buffer.as_ptr(), dest_ptr, copy_len); } } else { // Handle z/Z specially to store converted value @@ -641,43 +641,43 @@ impl PyCPointer { // Multi-byte types need read_unaligned for safety on strict-alignment architectures Some("h") => Ok(vm .ctx - .new_int(std::ptr::read_unaligned(ptr as *const i16) as i32) + .new_int(core::ptr::read_unaligned(ptr as *const i16) as i32) .into()), Some("H") => Ok(vm .ctx - .new_int(std::ptr::read_unaligned(ptr as *const u16) as i32) + .new_int(core::ptr::read_unaligned(ptr as *const u16) as i32) .into()), Some("i") | Some("l") => Ok(vm .ctx - .new_int(std::ptr::read_unaligned(ptr as *const i32)) + .new_int(core::ptr::read_unaligned(ptr as *const i32)) .into()), Some("I") | Some("L") => Ok(vm .ctx - .new_int(std::ptr::read_unaligned(ptr as *const u32)) + .new_int(core::ptr::read_unaligned(ptr as *const u32)) .into()), Some("q") => Ok(vm .ctx - .new_int(std::ptr::read_unaligned(ptr as *const i64)) + .new_int(core::ptr::read_unaligned(ptr as *const i64)) .into()), Some("Q") => Ok(vm .ctx - .new_int(std::ptr::read_unaligned(ptr as *const u64)) + .new_int(core::ptr::read_unaligned(ptr as *const u64)) .into()), Some("f") => Ok(vm .ctx - .new_float(std::ptr::read_unaligned(ptr as *const f32) as f64) + .new_float(core::ptr::read_unaligned(ptr as *const f32) as f64) .into()), Some("d") | Some("g") => Ok(vm .ctx - .new_float(std::ptr::read_unaligned(ptr as *const f64)) + .new_float(core::ptr::read_unaligned(ptr as *const f64)) .into()), Some("P") | Some("z") | Some("Z") => Ok(vm .ctx - .new_int(std::ptr::read_unaligned(ptr as *const usize)) + .new_int(core::ptr::read_unaligned(ptr as *const usize)) .into()), _ => { // Default: read as bytes - let bytes = std::slice::from_raw_parts(ptr, size).to_vec(); + let bytes = core::slice::from_raw_parts(ptr, size).to_vec(); Ok(vm.ctx.new_bytes(bytes).into()) } } @@ -708,7 +708,7 @@ impl PyCPointer { "bytes/string or integer address expected".to_owned(), )); }; - std::ptr::write_unaligned(ptr as *mut usize, ptr_val); + core::ptr::write_unaligned(ptr as *mut usize, ptr_val); return Ok(()); } _ => {} @@ -723,19 +723,19 @@ impl PyCPointer { *ptr = i.to_u8().expect("int too large"); } 2 => { - std::ptr::write_unaligned( + core::ptr::write_unaligned( ptr as *mut i16, i.to_i16().expect("int too large"), ); } 4 => { - std::ptr::write_unaligned( + core::ptr::write_unaligned( ptr as *mut i32, i.to_i32().expect("int too large"), ); } 8 => { - std::ptr::write_unaligned( + core::ptr::write_unaligned( ptr as *mut i64, i.to_i64().expect("int too large"), ); @@ -743,7 +743,7 @@ impl PyCPointer { _ => { let bytes = i.to_signed_bytes_le(); let copy_len = bytes.len().min(size); - std::ptr::copy_nonoverlapping(bytes.as_ptr(), ptr, copy_len); + core::ptr::copy_nonoverlapping(bytes.as_ptr(), ptr, copy_len); } } return Ok(()); @@ -754,10 +754,10 @@ impl PyCPointer { let f = float_val.to_f64(); match size { 4 => { - std::ptr::write_unaligned(ptr as *mut f32, f as f32); + core::ptr::write_unaligned(ptr as *mut f32, f as f32); } 8 => { - std::ptr::write_unaligned(ptr as *mut f64, f); + core::ptr::write_unaligned(ptr as *mut f64, f); } _ => {} } @@ -767,7 +767,7 @@ impl PyCPointer { // Try bytes if let Ok(bytes) = value.try_bytes_like(vm, |b| b.to_vec()) { let copy_len = bytes.len().min(size); - std::ptr::copy_nonoverlapping(bytes.as_ptr(), ptr, copy_len); + core::ptr::copy_nonoverlapping(bytes.as_ptr(), ptr, copy_len); return Ok(()); } diff --git a/crates/vm/src/stdlib/ctypes/simple.rs b/crates/vm/src/stdlib/ctypes/simple.rs index fbb17620fe4..9835953812f 100644 --- a/crates/vm/src/stdlib/ctypes/simple.rs +++ b/crates/vm/src/stdlib/ctypes/simple.rs @@ -13,9 +13,9 @@ use crate::function::{Either, FuncArgs, OptionalArg}; use crate::protocol::{BufferDescriptor, PyBuffer, PyNumberMethods}; use crate::types::{AsBuffer, AsNumber, Constructor, Initializer, Representable}; use crate::{AsObject, Py, PyObject, PyObjectRef, PyPayload, PyRef, PyResult, VirtualMachine}; +use alloc::borrow::Cow; +use core::fmt::Debug; use num_traits::ToPrimitive; -use std::borrow::Cow; -use std::fmt::Debug; /// Valid type codes for ctypes simple types // spell-checker: disable-next-line @@ -27,22 +27,22 @@ pub(super) const SIMPLE_TYPE_CHARS: &str = "cbBhHiIlLdfuzZqQPXOv?g"; fn ctypes_code_to_pep3118(code: char) -> char { match code { // c_int: map based on sizeof(int) - 'i' if std::mem::size_of::() == 2 => 'h', - 'i' if std::mem::size_of::() == 4 => 'i', - 'i' if std::mem::size_of::() == 8 => 'q', - 'I' if std::mem::size_of::() == 2 => 'H', - 'I' if std::mem::size_of::() == 4 => 'I', - 'I' if std::mem::size_of::() == 8 => 'Q', + 'i' if core::mem::size_of::() == 2 => 'h', + 'i' if core::mem::size_of::() == 4 => 'i', + 'i' if core::mem::size_of::() == 8 => 'q', + 'I' if core::mem::size_of::() == 2 => 'H', + 'I' if core::mem::size_of::() == 4 => 'I', + 'I' if core::mem::size_of::() == 8 => 'Q', // c_long: map based on sizeof(long) - 'l' if std::mem::size_of::() == 4 => 'l', - 'l' if std::mem::size_of::() == 8 => 'q', - 'L' if std::mem::size_of::() == 4 => 'L', - 'L' if std::mem::size_of::() == 8 => 'Q', + 'l' if core::mem::size_of::() == 4 => 'l', + 'l' if core::mem::size_of::() == 8 => 'q', + 'L' if core::mem::size_of::() == 4 => 'L', + 'L' if core::mem::size_of::() == 8 => 'Q', // c_bool: map based on sizeof(bool) - typically 1 byte on all platforms - '?' if std::mem::size_of::() == 1 => '?', - '?' if std::mem::size_of::() == 2 => 'H', - '?' if std::mem::size_of::() == 4 => 'L', - '?' if std::mem::size_of::() == 8 => 'Q', + '?' if core::mem::size_of::() == 1 => '?', + '?' if core::mem::size_of::() == 2 => 'H', + '?' if core::mem::size_of::() == 4 => 'L', + '?' if core::mem::size_of::() == 8 => 'Q', // Default: use the same code _ => code, } @@ -268,7 +268,7 @@ impl PyCSimpleType { let create_simple_with_value = |type_str: &str, val: &PyObject| -> PyResult { let simple = new_simple_type(Either::B(&cls), vm)?; let buffer_bytes = value_to_bytes_endian(type_str, val, false, vm); - *simple.0.buffer.write() = std::borrow::Cow::Owned(buffer_bytes.clone()); + *simple.0.buffer.write() = alloc::borrow::Cow::Owned(buffer_bytes.clone()); let simple_obj: PyObjectRef = simple.into_ref_with_type(vm, cls.clone())?.into(); // from_param returns CArgObject, not the simple type itself let tag = type_str.as_bytes().first().copied().unwrap_or(b'?'); @@ -418,9 +418,9 @@ impl PyCSimpleType { if let Some(funcptr) = value.downcast_ref::() { let ptr_val = { let buffer = funcptr._base.buffer.read(); - if buffer.len() >= std::mem::size_of::() { + if buffer.len() >= core::mem::size_of::() { usize::from_ne_bytes( - buffer[..std::mem::size_of::()].try_into().unwrap(), + buffer[..core::mem::size_of::()].try_into().unwrap(), ) } else { 0 @@ -441,9 +441,9 @@ impl PyCSimpleType { if matches!(value_type_code.as_deref(), Some("z") | Some("Z")) { let ptr_val = { let buffer = simple.0.buffer.read(); - if buffer.len() >= std::mem::size_of::() { + if buffer.len() >= core::mem::size_of::() { usize::from_ne_bytes( - buffer[..std::mem::size_of::()].try_into().unwrap(), + buffer[..core::mem::size_of::()].try_into().unwrap(), ) } else { 0 @@ -712,7 +712,7 @@ fn create_swapped_types( pub struct PyCSimple(pub PyCData); impl Debug for PyCSimple { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result { f.debug_struct("PyCSimple") .field("size", &self.0.buffer.read().len()) .finish() @@ -833,7 +833,7 @@ fn value_to_bytes_endian( let v = int_val.as_bigint().to_i128().expect("int too large") as libc::c_long; return to_bytes!(v); } - const SIZE: usize = std::mem::size_of::(); + const SIZE: usize = core::mem::size_of::(); vec![0; SIZE] } "L" => { @@ -842,7 +842,7 @@ fn value_to_bytes_endian( let v = int_val.as_bigint().to_i128().expect("int too large") as libc::c_ulong; return to_bytes!(v); } - const SIZE: usize = std::mem::size_of::(); + const SIZE: usize = core::mem::size_of::(); vec![0; SIZE] } "q" => { @@ -938,7 +938,7 @@ fn value_to_bytes_endian( .expect("int too large for pointer"); return to_bytes!(v); } - vec![0; std::mem::size_of::()] + vec![0; core::mem::size_of::()] } "z" => { // c_char_p - pointer to char (stores pointer value from int) @@ -950,7 +950,7 @@ fn value_to_bytes_endian( .expect("int too large for pointer"); return to_bytes!(v); } - vec![0; std::mem::size_of::()] + vec![0; core::mem::size_of::()] } "Z" => { // c_wchar_p - pointer to wchar_t (stores pointer value from int) @@ -962,7 +962,7 @@ fn value_to_bytes_endian( .expect("int too large for pointer"); return to_bytes!(v); } - vec![0; std::mem::size_of::()] + vec![0; core::mem::size_of::()] } "O" => { // py_object - store object id as non-zero marker @@ -1166,7 +1166,7 @@ impl PyCSimple { } // Read null-terminated string at the address unsafe { - let cstr = std::ffi::CStr::from_ptr(ptr as _); + let cstr = core::ffi::CStr::from_ptr(ptr as _); return Ok(vm.ctx.new_bytes(cstr.to_bytes().to_vec()).into()); } } @@ -1183,7 +1183,7 @@ impl PyCSimple { unsafe { let w_ptr = ptr as *const libc::wchar_t; let len = libc::wcslen(w_ptr); - let wchars = std::slice::from_raw_parts(w_ptr, len); + let wchars = core::slice::from_raw_parts(w_ptr, len); #[cfg(windows)] { use rustpython_common::wtf8::Wtf8Buf; @@ -1221,13 +1221,13 @@ impl PyCSimple { // Read value from buffer, swap bytes if needed let buffer = zelf.0.buffer.read(); - let buffer_data: std::borrow::Cow<'_, [u8]> = if swapped { + let buffer_data: alloc::borrow::Cow<'_, [u8]> = if swapped { // Reverse bytes for swapped endian types let mut swapped_bytes = buffer.to_vec(); swapped_bytes.reverse(); - std::borrow::Cow::Owned(swapped_bytes) + alloc::borrow::Cow::Owned(swapped_bytes) } else { - std::borrow::Cow::Borrowed(&*buffer) + alloc::borrow::Cow::Borrowed(&*buffer) }; let cls_ref = cls.to_owned(); @@ -1269,7 +1269,7 @@ impl PyCSimple { if type_code == "z" { if let Some(bytes) = value.downcast_ref::() { let (converted, ptr) = super::base::ensure_z_null_terminated(bytes, vm); - *zelf.0.buffer.write() = std::borrow::Cow::Owned(ptr.to_ne_bytes().to_vec()); + *zelf.0.buffer.write() = alloc::borrow::Cow::Owned(ptr.to_ne_bytes().to_vec()); *zelf.0.objects.write() = Some(converted); return Ok(()); } @@ -1277,7 +1277,7 @@ impl PyCSimple { && let Some(s) = value.downcast_ref::() { let (holder, ptr) = super::base::str_to_wchar_bytes(s.as_str(), vm); - *zelf.0.buffer.write() = std::borrow::Cow::Owned(ptr.to_ne_bytes().to_vec()); + *zelf.0.buffer.write() = alloc::borrow::Cow::Owned(ptr.to_ne_bytes().to_vec()); *zelf.0.objects.write() = Some(holder); return Ok(()); } @@ -1368,65 +1368,65 @@ impl PyCSimple { let buffer = self.0.buffer.read(); let bytes: &[u8] = &buffer; - if std::ptr::eq(ty.as_raw_ptr(), libffi::middle::Type::u8().as_raw_ptr()) { + if core::ptr::eq(ty.as_raw_ptr(), libffi::middle::Type::u8().as_raw_ptr()) { if !bytes.is_empty() { return Some(FfiArgValue::U8(bytes[0])); } - } else if std::ptr::eq(ty.as_raw_ptr(), libffi::middle::Type::i8().as_raw_ptr()) { + } else if core::ptr::eq(ty.as_raw_ptr(), libffi::middle::Type::i8().as_raw_ptr()) { if !bytes.is_empty() { return Some(FfiArgValue::I8(bytes[0] as i8)); } - } else if std::ptr::eq(ty.as_raw_ptr(), libffi::middle::Type::u16().as_raw_ptr()) { + } else if core::ptr::eq(ty.as_raw_ptr(), libffi::middle::Type::u16().as_raw_ptr()) { if bytes.len() >= 2 { return Some(FfiArgValue::U16(u16::from_ne_bytes([bytes[0], bytes[1]]))); } - } else if std::ptr::eq(ty.as_raw_ptr(), libffi::middle::Type::i16().as_raw_ptr()) { + } else if core::ptr::eq(ty.as_raw_ptr(), libffi::middle::Type::i16().as_raw_ptr()) { if bytes.len() >= 2 { return Some(FfiArgValue::I16(i16::from_ne_bytes([bytes[0], bytes[1]]))); } - } else if std::ptr::eq(ty.as_raw_ptr(), libffi::middle::Type::u32().as_raw_ptr()) { + } else if core::ptr::eq(ty.as_raw_ptr(), libffi::middle::Type::u32().as_raw_ptr()) { if bytes.len() >= 4 { return Some(FfiArgValue::U32(u32::from_ne_bytes([ bytes[0], bytes[1], bytes[2], bytes[3], ]))); } - } else if std::ptr::eq(ty.as_raw_ptr(), libffi::middle::Type::i32().as_raw_ptr()) { + } else if core::ptr::eq(ty.as_raw_ptr(), libffi::middle::Type::i32().as_raw_ptr()) { if bytes.len() >= 4 { return Some(FfiArgValue::I32(i32::from_ne_bytes([ bytes[0], bytes[1], bytes[2], bytes[3], ]))); } - } else if std::ptr::eq(ty.as_raw_ptr(), libffi::middle::Type::u64().as_raw_ptr()) { + } else if core::ptr::eq(ty.as_raw_ptr(), libffi::middle::Type::u64().as_raw_ptr()) { if bytes.len() >= 8 { return Some(FfiArgValue::U64(u64::from_ne_bytes([ bytes[0], bytes[1], bytes[2], bytes[3], bytes[4], bytes[5], bytes[6], bytes[7], ]))); } - } else if std::ptr::eq(ty.as_raw_ptr(), libffi::middle::Type::i64().as_raw_ptr()) { + } else if core::ptr::eq(ty.as_raw_ptr(), libffi::middle::Type::i64().as_raw_ptr()) { if bytes.len() >= 8 { return Some(FfiArgValue::I64(i64::from_ne_bytes([ bytes[0], bytes[1], bytes[2], bytes[3], bytes[4], bytes[5], bytes[6], bytes[7], ]))); } - } else if std::ptr::eq(ty.as_raw_ptr(), libffi::middle::Type::f32().as_raw_ptr()) { + } else if core::ptr::eq(ty.as_raw_ptr(), libffi::middle::Type::f32().as_raw_ptr()) { if bytes.len() >= 4 { return Some(FfiArgValue::F32(f32::from_ne_bytes([ bytes[0], bytes[1], bytes[2], bytes[3], ]))); } - } else if std::ptr::eq(ty.as_raw_ptr(), libffi::middle::Type::f64().as_raw_ptr()) { + } else if core::ptr::eq(ty.as_raw_ptr(), libffi::middle::Type::f64().as_raw_ptr()) { if bytes.len() >= 8 { return Some(FfiArgValue::F64(f64::from_ne_bytes([ bytes[0], bytes[1], bytes[2], bytes[3], bytes[4], bytes[5], bytes[6], bytes[7], ]))); } - } else if std::ptr::eq( + } else if core::ptr::eq( ty.as_raw_ptr(), libffi::middle::Type::pointer().as_raw_ptr(), - ) && bytes.len() >= std::mem::size_of::() + ) && bytes.len() >= core::mem::size_of::() { let val = - usize::from_ne_bytes(bytes[..std::mem::size_of::()].try_into().unwrap()); + usize::from_ne_bytes(bytes[..core::mem::size_of::()].try_into().unwrap()); return Some(FfiArgValue::Pointer(val)); } None diff --git a/crates/vm/src/stdlib/ctypes/structure.rs b/crates/vm/src/stdlib/ctypes/structure.rs index d5aca392c52..295ce0d87cf 100644 --- a/crates/vm/src/stdlib/ctypes/structure.rs +++ b/crates/vm/src/stdlib/ctypes/structure.rs @@ -6,9 +6,9 @@ use crate::function::PySetterValue; use crate::protocol::{BufferDescriptor, PyBuffer, PyNumberMethods}; use crate::types::{AsBuffer, AsNumber, Constructor, Initializer, SetAttr}; use crate::{AsObject, Py, PyObjectRef, PyPayload, PyResult, VirtualMachine}; +use alloc::borrow::Cow; +use core::fmt::Debug; use num_traits::ToPrimitive; -use std::borrow::Cow; -use std::fmt::Debug; /// Calculate Structure type size from _fields_ (sum of field sizes) pub(super) fn calculate_struct_size(cls: &Py, vm: &VirtualMachine) -> PyResult { @@ -206,7 +206,7 @@ impl PyCStructType { { ( baseinfo.size, - std::cmp::max(baseinfo.align, forced_alignment), + core::cmp::max(baseinfo.align, forced_alignment), baseinfo.flags.contains(StgInfoFlags::TYPEFLAG_HASPOINTER), baseinfo.flags.contains(StgInfoFlags::TYPEFLAG_HASUNION), baseinfo.flags.contains(StgInfoFlags::TYPEFLAG_HASBITFIELD), @@ -252,7 +252,7 @@ impl PyCStructType { // Calculate effective alignment (PyCField_FromDesc) let effective_align = if pack > 0 { - std::cmp::min(pack, field_align) + core::cmp::min(pack, field_align) } else { field_align }; @@ -347,7 +347,7 @@ impl PyCStructType { } // Calculate total_align = max(max_align, forced_alignment) - let total_align = std::cmp::max(max_align, forced_alignment); + let total_align = core::cmp::max(max_align, forced_alignment); // Calculate aligned_size (PyCStructUnionType_update_stginfo) let aligned_size = if total_align > 0 { @@ -501,7 +501,7 @@ impl SetAttr for PyCStructType { pub struct PyCStructure(pub PyCData); impl Debug for PyCStructure { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result { f.debug_struct("PyCStructure") .field("size", &self.0.size()) .finish() diff --git a/crates/vm/src/stdlib/ctypes/union.rs b/crates/vm/src/stdlib/ctypes/union.rs index 41bc7492a25..0da1ffee3fd 100644 --- a/crates/vm/src/stdlib/ctypes/union.rs +++ b/crates/vm/src/stdlib/ctypes/union.rs @@ -7,7 +7,7 @@ use crate::function::PySetterValue; use crate::protocol::{BufferDescriptor, PyBuffer}; use crate::types::{AsBuffer, Constructor, Initializer, SetAttr}; use crate::{AsObject, Py, PyObjectRef, PyPayload, PyResult, VirtualMachine}; -use std::borrow::Cow; +use alloc::borrow::Cow; /// Calculate Union type size from _fields_ (max field size) pub(super) fn calculate_union_size(cls: &Py, vm: &VirtualMachine) -> PyResult { @@ -175,7 +175,7 @@ impl PyCUnionType { { ( baseinfo.size, - std::cmp::max(baseinfo.align, forced_alignment), + core::cmp::max(baseinfo.align, forced_alignment), baseinfo.flags.contains(StgInfoFlags::TYPEFLAG_HASPOINTER), baseinfo.flags.contains(StgInfoFlags::TYPEFLAG_HASBITFIELD), baseinfo.ffi_field_types.clone(), @@ -215,7 +215,7 @@ impl PyCUnionType { // Calculate effective alignment let effective_align = if pack > 0 { - std::cmp::min(pack, field_align) + core::cmp::min(pack, field_align) } else { field_align }; @@ -264,7 +264,7 @@ impl PyCUnionType { } // Calculate total_align and aligned_size - let total_align = std::cmp::max(max_align, forced_alignment); + let total_align = core::cmp::max(max_align, forced_alignment); let aligned_size = if total_align > 0 { max_size.div_ceil(total_align) * total_align } else { @@ -418,8 +418,8 @@ impl SetAttr for PyCUnionType { #[repr(transparent)] pub struct PyCUnion(pub PyCData); -impl std::fmt::Debug for PyCUnion { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { +impl core::fmt::Debug for PyCUnion { + fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result { f.debug_struct("PyCUnion") .field("size", &self.0.size()) .finish() diff --git a/crates/vm/src/stdlib/io.rs b/crates/vm/src/stdlib/io.rs index 52900faec08..54a38ef20e6 100644 --- a/crates/vm/src/stdlib/io.rs +++ b/crates/vm/src/stdlib/io.rs @@ -53,7 +53,7 @@ impl ToOSErrorBuilder for std::io::Error { let msg = { let ptr = unsafe { libc::strerror(errno) }; if !ptr.is_null() { - unsafe { std::ffi::CStr::from_ptr(ptr) } + unsafe { core::ffi::CStr::from_ptr(ptr) } .to_string_lossy() .into_owned() } else { @@ -183,16 +183,16 @@ mod _io { }, vm::VirtualMachine, }; + use alloc::borrow::Cow; use bstr::ByteSlice; - use crossbeam_utils::atomic::AtomicCell; - use malachite_bigint::BigInt; - use num_traits::ToPrimitive; - use std::{ - borrow::Cow, - io::{self, Cursor, SeekFrom, prelude::*}, + use core::{ ops::Range, sync::atomic::{AtomicBool, Ordering}, }; + use crossbeam_utils::atomic::AtomicCell; + use malachite_bigint::BigInt; + use num_traits::ToPrimitive; + use std::io::{self, Cursor, SeekFrom, prelude::*}; #[allow(clippy::let_and_return)] fn validate_whence(whence: i32) -> bool { @@ -354,7 +354,7 @@ mod _io { // if we don't specify the number of bytes, or it's too big, give the whole rest of the slice let n = bytes.map_or_else( || avail_slice.len(), - |n| std::cmp::min(n, avail_slice.len()), + |n| core::cmp::min(n, avail_slice.len()), ); let b = avail_slice[..n].to_vec(); self.cursor.set_position((pos + n) as u64); @@ -1059,7 +1059,7 @@ mod _io { // TODO: loop if write() raises an interrupt vm.call_method(self.raw.as_ref().unwrap(), "write", (mem_obj,))? } else { - let v = std::mem::take(&mut self.buffer); + let v = core::mem::take(&mut self.buffer); let write_buf = VecBuffer::from(v).into_ref(&vm.ctx); let mem_obj = PyMemoryView::from_buffer_range( write_buf.clone().into_pybuffer(true), @@ -1330,7 +1330,7 @@ mod _io { let res = match v { Either::A(v) => { let v = v.unwrap_or(&mut self.buffer); - let read_buf = VecBuffer::from(std::mem::take(v)).into_ref(&vm.ctx); + let read_buf = VecBuffer::from(core::mem::take(v)).into_ref(&vm.ctx); let mem_obj = PyMemoryView::from_buffer_range( read_buf.clone().into_pybuffer(false), buf_range, @@ -1527,7 +1527,7 @@ mod _io { } else if !(readinto1 && written != 0) { let n = self.fill_buffer(vm)?; if let Some(n) = n.filter(|&n| n > 0) { - let n = std::cmp::min(n, remaining); + let n = core::cmp::min(n, remaining); buf.as_contiguous_mut().unwrap()[written..][..n] .copy_from_slice(&self.buffer[self.pos as usize..][..n]); self.pos += n as Offset; @@ -1881,7 +1881,7 @@ mod _io { } let have = data.readahead(); if have > 0 { - let n = std::cmp::min(have as usize, n); + let n = core::cmp::min(have as usize, n); return Ok(data.read_fast(n).unwrap()); } // Flush write buffer before reading @@ -2373,7 +2373,7 @@ mod _io { } } - impl std::ops::Add for Utf8size { + impl core::ops::Add for Utf8size { type Output = Self; #[inline] @@ -2383,7 +2383,7 @@ mod _io { } } - impl std::ops::AddAssign for Utf8size { + impl core::ops::AddAssign for Utf8size { #[inline] fn add_assign(&mut self, rhs: Self) { self.bytes += rhs.bytes; @@ -2391,7 +2391,7 @@ mod _io { } } - impl std::ops::Sub for Utf8size { + impl core::ops::Sub for Utf8size { type Output = Self; #[inline] @@ -2401,7 +2401,7 @@ mod _io { } } - impl std::ops::SubAssign for Utf8size { + impl core::ops::SubAssign for Utf8size { #[inline] fn sub_assign(&mut self, rhs: Self) { self.bytes -= rhs.bytes; @@ -2470,7 +2470,7 @@ mod _io { impl PendingWrites { fn push(&mut self, write: PendingWrite) { self.num_bytes += write.as_bytes().len(); - self.data = match std::mem::take(&mut self.data) { + self.data = match core::mem::take(&mut self.data) { PendingWritesData::None => PendingWritesData::One(write), PendingWritesData::One(write1) => PendingWritesData::Many(vec![write1, write]), PendingWritesData::Many(mut v) => { @@ -2480,13 +2480,13 @@ mod _io { } } fn take(&mut self, vm: &VirtualMachine) -> PyBytesRef { - let Self { num_bytes, data } = std::mem::take(self); + let Self { num_bytes, data } = core::mem::take(self); if let PendingWritesData::One(PendingWrite::Bytes(b)) = data { return b; } let writes_iter = match data { PendingWritesData::None => itertools::Either::Left(vec![].into_iter()), - PendingWritesData::One(write) => itertools::Either::Right(std::iter::once(write)), + PendingWritesData::One(write) => itertools::Either::Right(core::iter::once(write)), PendingWritesData::Many(writes) => itertools::Either::Left(writes.into_iter()), }; let mut buf = Vec::with_capacity(num_bytes); @@ -2508,7 +2508,7 @@ mod _io { impl TextIOCookie { const START_POS_OFF: usize = 0; - const DEC_FLAGS_OFF: usize = Self::START_POS_OFF + std::mem::size_of::(); + const DEC_FLAGS_OFF: usize = Self::START_POS_OFF + core::mem::size_of::(); const BYTES_TO_FEED_OFF: usize = Self::DEC_FLAGS_OFF + 4; const CHARS_TO_SKIP_OFF: usize = Self::BYTES_TO_FEED_OFF + 4; const NEED_EOF_OFF: usize = Self::CHARS_TO_SKIP_OFF + 4; @@ -2525,7 +2525,7 @@ mod _io { macro_rules! get_field { ($t:ty, $off:ident) => {{ <$t>::from_ne_bytes( - buf[Self::$off..][..std::mem::size_of::<$t>()] + buf[Self::$off..][..core::mem::size_of::<$t>()] .try_into() .unwrap(), ) @@ -2546,7 +2546,7 @@ mod _io { macro_rules! set_field { ($field:expr, $off:ident) => {{ let field = $field; - buf[Self::$off..][..std::mem::size_of_val(&field)] + buf[Self::$off..][..core::mem::size_of_val(&field)] .copy_from_slice(&field.to_ne_bytes()) }}; } @@ -3509,7 +3509,7 @@ mod _io { } else { size_hint }; - let chunk_size = std::cmp::max(self.chunk_size, size_hint); + let chunk_size = core::cmp::max(self.chunk_size, size_hint); let input_chunk = vm.call_method(&self.buffer, method, (chunk_size,))?; let buf = ArgBytesLike::try_from_borrowed_object(vm, &input_chunk).map_err(|_| { @@ -3591,8 +3591,8 @@ mod _io { vm: &VirtualMachine, ) -> PyStrRef { let empty_str = || vm.ctx.empty_str.to_owned(); - let chars_pos = std::mem::take(&mut self.decoded_chars_used).bytes; - let decoded_chars = match std::mem::take(&mut self.decoded_chars) { + let chars_pos = core::mem::take(&mut self.decoded_chars_used).bytes; + let decoded_chars = match core::mem::take(&mut self.decoded_chars) { None => return append.unwrap_or_else(empty_str), Some(s) if s.is_empty() => return append.unwrap_or_else(empty_str), Some(s) => s, @@ -4294,7 +4294,7 @@ mod _io { plus: bool, } - impl std::str::FromStr for Mode { + impl core::str::FromStr for Mode { type Err = ParseModeError; fn from_str(s: &str) -> Result { diff --git a/crates/vm/src/stdlib/itertools.rs b/crates/vm/src/stdlib/itertools.rs index 3fedd17f12b..3aad2f91931 100644 --- a/crates/vm/src/stdlib/itertools.rs +++ b/crates/vm/src/stdlib/itertools.rs @@ -25,8 +25,8 @@ mod decl { use malachite_bigint::BigInt; use num_traits::One; + use alloc::fmt; use num_traits::{Signed, ToPrimitive}; - use std::fmt; fn pickle_deprecation(vm: &VirtualMachine) -> PyResult<()> { warnings::warn( @@ -1320,7 +1320,7 @@ mod decl { for arg in iterables.iter() { pools.push(arg.try_to_value(vm)?); } - let pools = std::iter::repeat_n(pools, repeat) + let pools = core::iter::repeat_n(pools, repeat) .flatten() .collect::>>(); diff --git a/crates/vm/src/stdlib/mod.rs b/crates/vm/src/stdlib/mod.rs index 9fae516fe04..e46f333a28b 100644 --- a/crates/vm/src/stdlib/mod.rs +++ b/crates/vm/src/stdlib/mod.rs @@ -62,7 +62,8 @@ mod winapi; mod winreg; use crate::{PyRef, VirtualMachine, builtins::PyModule}; -use std::{borrow::Cow, collections::HashMap}; +use alloc::borrow::Cow; +use std::collections::HashMap; pub type StdlibInitFunc = Box PyRef)>; pub type StdlibMap = HashMap, StdlibInitFunc, ahash::RandomState>; diff --git a/crates/vm/src/stdlib/nt.rs b/crates/vm/src/stdlib/nt.rs index e056142658d..a32959808c0 100644 --- a/crates/vm/src/stdlib/nt.rs +++ b/crates/vm/src/stdlib/nt.rs @@ -497,7 +497,7 @@ pub(crate) mod module { wide_path.as_ptr(), FILE_READ_ATTRIBUTES, FILE_SHARE_READ | FILE_SHARE_WRITE | FILE_SHARE_DELETE, - std::ptr::null(), + core::ptr::null(), OPEN_EXISTING, flags, std::ptr::null_mut(), @@ -517,7 +517,7 @@ pub(crate) mod module { wide_path.as_ptr(), FILE_READ_ATTRIBUTES, FILE_SHARE_READ | FILE_SHARE_WRITE | FILE_SHARE_DELETE, - std::ptr::null(), + core::ptr::null(), OPEN_EXISTING, FILE_FLAG_OPEN_REPARSE_POINT, std::ptr::null_mut(), @@ -568,7 +568,7 @@ pub(crate) mod module { wide_path.as_ptr(), FILE_READ_ATTRIBUTES, FILE_SHARE_READ | FILE_SHARE_WRITE | FILE_SHARE_DELETE, - std::ptr::null(), + core::ptr::null(), OPEN_EXISTING, flags, std::ptr::null_mut(), @@ -586,7 +586,7 @@ pub(crate) mod module { wide_path.as_ptr(), GENERIC_READ, FILE_SHARE_READ | FILE_SHARE_WRITE, - std::ptr::null(), + core::ptr::null(), OPEN_EXISTING, 0, std::ptr::null_mut(), @@ -733,7 +733,7 @@ pub(crate) mod module { volume.as_ptr(), FILE_READ_ATTRIBUTES, FILE_SHARE_READ | FILE_SHARE_WRITE, - std::ptr::null(), + core::ptr::null(), OPEN_EXISTING, FILE_FLAG_BACKUP_SEMANTICS, std::ptr::null_mut(), @@ -862,7 +862,7 @@ pub(crate) mod module { conout.as_ptr(), Foundation::GENERIC_READ | Foundation::GENERIC_WRITE, FileSystem::FILE_SHARE_READ | FileSystem::FILE_SHARE_WRITE, - std::ptr::null(), + core::ptr::null(), FileSystem::OPEN_EXISTING, 0, std::ptr::null_mut(), @@ -933,7 +933,7 @@ pub(crate) mod module { let argv_spawn: Vec<*const u16> = argv .iter() .map(|v| v.as_ptr()) - .chain(once(std::ptr::null())) + .chain(once(core::ptr::null())) .collect(); let result = unsafe { suppress_iph!(_wspawnv(mode, path.as_ptr(), argv_spawn.as_ptr())) }; @@ -976,7 +976,7 @@ pub(crate) mod module { let argv_spawn: Vec<*const u16> = argv .iter() .map(|v| v.as_ptr()) - .chain(once(std::ptr::null())) + .chain(once(core::ptr::null())) .collect(); // Build environment strings as "KEY=VALUE\0" wide strings @@ -1004,7 +1004,7 @@ pub(crate) mod module { let envp: Vec<*const u16> = env_strings .iter() .map(|s| s.as_ptr()) - .chain(once(std::ptr::null())) + .chain(once(core::ptr::null())) .collect(); let result = unsafe { @@ -1052,7 +1052,7 @@ pub(crate) mod module { let argv_execv: Vec<*const u16> = argv .iter() .map(|v| v.as_ptr()) - .chain(once(std::ptr::null())) + .chain(once(core::ptr::null())) .collect(); if (unsafe { suppress_iph!(_wexecv(path.as_ptr(), argv_execv.as_ptr())) } == -1) { @@ -1093,7 +1093,7 @@ pub(crate) mod module { let argv_execve: Vec<*const u16> = argv .iter() .map(|v| v.as_ptr()) - .chain(once(std::ptr::null())) + .chain(once(core::ptr::null())) .collect(); // Build environment strings as "KEY=VALUE\0" wide strings @@ -1121,7 +1121,7 @@ pub(crate) mod module { let envp: Vec<*const u16> = env_strings .iter() .map(|s| s.as_ptr()) - .chain(once(std::ptr::null())) + .chain(once(core::ptr::null())) .collect(); if (unsafe { suppress_iph!(_wexecve(path.as_ptr(), argv_execve.as_ptr(), envp.as_ptr())) } @@ -1356,7 +1356,7 @@ pub(crate) mod module { .chain(std::iter::once(0)) // null-terminated .collect(); - let mut end: *const u16 = std::ptr::null(); + let mut end: *const u16 = core::ptr::null(); let hr = unsafe { windows_sys::Win32::UI::Shell::PathCchSkipRoot(backslashed.as_ptr(), &mut end) }; @@ -1667,7 +1667,7 @@ pub(crate) mod module { let res = CreatePipe( read.as_mut_ptr() as *mut _, write.as_mut_ptr() as *mut _, - std::ptr::null(), + core::ptr::null(), 0, ); if res == 0 { @@ -1723,7 +1723,7 @@ pub(crate) mod module { let Some(func) = func else { return 0; }; - let nt_query: NtQueryInformationProcessFn = unsafe { std::mem::transmute(func) }; + let nt_query: NtQueryInformationProcessFn = unsafe { core::mem::transmute(func) }; let mut info: PROCESS_BASIC_INFORMATION = unsafe { std::mem::zeroed() }; @@ -1808,7 +1808,7 @@ pub(crate) mod module { wide_path.as_ptr(), 0, // No access needed, just reading reparse data FILE_SHARE_READ | FILE_SHARE_WRITE | FILE_SHARE_DELETE, - std::ptr::null(), + core::ptr::null(), OPEN_EXISTING, FILE_FLAG_BACKUP_SEMANTICS | FILE_FLAG_OPEN_REPARSE_POINT, std::ptr::null_mut(), @@ -1832,7 +1832,7 @@ pub(crate) mod module { DeviceIoControl( handle, FSCTL_GET_REPARSE_POINT, - std::ptr::null(), + core::ptr::null(), 0, buffer.as_mut_ptr() as *mut _, BUFFER_SIZE as u32, diff --git a/crates/vm/src/stdlib/os.rs b/crates/vm/src/stdlib/os.rs index 6e2bb274fec..add8763f5e1 100644 --- a/crates/vm/src/stdlib/os.rs +++ b/crates/vm/src/stdlib/os.rs @@ -171,15 +171,10 @@ pub(super) mod _os { utils::ToCString, vm::VirtualMachine, }; + use core::time::Duration; use crossbeam_utils::atomic::AtomicCell; use itertools::Itertools; - use std::{ - env, fs, - fs::OpenOptions, - io, - path::PathBuf, - time::{Duration, SystemTime}, - }; + use std::{env, fs, fs::OpenOptions, io, path::PathBuf, time::SystemTime}; const OPEN_DIR_FD: bool = cfg!(not(any(windows, target_os = "redox"))); pub(crate) const MKDIR_DIR_FD: bool = cfg!(not(any(windows, target_os = "redox"))); @@ -518,7 +513,7 @@ pub(super) mod _os { 22, format!( "Invalid argument: {}", - std::str::from_utf8(key).unwrap_or("") + core::str::from_utf8(key).unwrap_or("") ), ); @@ -1051,12 +1046,12 @@ pub(super) mod _os { dir_fd: DirFd<'_, { STAT_DIR_FD as usize }>, follow_symlinks: FollowSymlinks, ) -> io::Result> { - let mut stat = std::mem::MaybeUninit::uninit(); + let mut stat = core::mem::MaybeUninit::uninit(); let ret = match file { OsPathOrFd::Path(path) => { use rustpython_common::os::ffi::OsStrExt; let path = path.as_ref().as_os_str().as_bytes(); - let path = match std::ffi::CString::new(path) { + let path = match alloc::ffi::CString::new(path) { Ok(x) => x, Err(_) => return Ok(None), }; @@ -1218,7 +1213,7 @@ pub(super) mod _os { use std::os::windows::io::AsRawHandle; use windows_sys::Win32::Storage::FileSystem; let handle = crt_fd::as_handle(fd).map_err(|e| e.into_pyexception(vm))?; - let mut distance_to_move: [i32; 2] = std::mem::transmute(position); + let mut distance_to_move: [i32; 2] = core::mem::transmute(position); let ret = FileSystem::SetFilePointer( handle.as_raw_handle(), distance_to_move[0], @@ -1229,7 +1224,7 @@ pub(super) mod _os { -1 } else { distance_to_move[0] = ret as _; - std::mem::transmute::<[i32; 2], i64>(distance_to_move) + core::mem::transmute::<[i32; 2], i64>(distance_to_move) } }; if res < 0 { @@ -1411,7 +1406,7 @@ pub(super) mod _os { .map_err(|err| OSErrorBuilder::with_filename(&err, path.clone(), vm))?; let ret = unsafe { - FileSystem::SetFileTime(f.as_raw_handle() as _, std::ptr::null(), &acc, &modif) + FileSystem::SetFileTime(f.as_raw_handle() as _, core::ptr::null(), &acc, &modif) }; if ret == 0 { @@ -1532,9 +1527,9 @@ pub(super) mod _os { #[pyfunction] fn copy_file_range(args: CopyFileRangeArgs<'_>, vm: &VirtualMachine) -> PyResult { #[allow(clippy::unnecessary_option_map_or_else)] - let p_offset_src = args.offset_src.as_ref().map_or_else(std::ptr::null, |x| x); + let p_offset_src = args.offset_src.as_ref().map_or_else(core::ptr::null, |x| x); #[allow(clippy::unnecessary_option_map_or_else)] - let p_offset_dst = args.offset_dst.as_ref().map_or_else(std::ptr::null, |x| x); + let p_offset_dst = args.offset_dst.as_ref().map_or_else(core::ptr::null, |x| x); let count: usize = args .count .try_into() @@ -1566,7 +1561,7 @@ pub(super) mod _os { #[pyfunction] fn strerror(e: i32) -> String { - unsafe { std::ffi::CStr::from_ptr(libc::strerror(e)) } + unsafe { core::ffi::CStr::from_ptr(libc::strerror(e)) } .to_string_lossy() .into_owned() } @@ -1670,7 +1665,7 @@ pub(super) mod _os { if encoding.is_null() || encoding.read() == '\0' as libc::c_char { "UTF-8".to_owned() } else { - std::ffi::CStr::from_ptr(encoding).to_string_lossy().into_owned() + core::ffi::CStr::from_ptr(encoding).to_string_lossy().into_owned() } }; diff --git a/crates/vm/src/stdlib/posix.rs b/crates/vm/src/stdlib/posix.rs index a4a311df06a..4ec71eecbf2 100644 --- a/crates/vm/src/stdlib/posix.rs +++ b/crates/vm/src/stdlib/posix.rs @@ -33,15 +33,15 @@ pub mod module { types::{Constructor, Representable}, utils::ToCString, }; + use alloc::ffi::CString; use bitflags::bitflags; + use core::ffi::CStr; use nix::{ fcntl, unistd::{self, Gid, Pid, Uid}, }; use std::{ - env, - ffi::{CStr, CString}, - fs, io, + env, fs, io, os::fd::{AsFd, BorrowedFd, FromRawFd, IntoRawFd, OwnedFd}, }; use strum_macros::{EnumIter, EnumString}; @@ -917,7 +917,7 @@ pub mod module { #[pyfunction] fn sched_getparam(pid: libc::pid_t, vm: &VirtualMachine) -> PyResult { let param = unsafe { - let mut param = std::mem::MaybeUninit::uninit(); + let mut param = core::mem::MaybeUninit::uninit(); if -1 == libc::sched_getparam(pid, param.as_mut_ptr()) { return Err(vm.new_last_errno_error()); } @@ -1280,7 +1280,7 @@ pub mod module { } fn try_from_id(vm: &VirtualMachine, obj: PyObjectRef, typ_name: &str) -> PyResult { - use std::cmp::Ordering; + use core::cmp::Ordering; let i = obj .try_to_ref::(vm) .map_err(|_| { @@ -1853,9 +1853,9 @@ pub mod module { #[pyfunction] fn dup2(args: Dup2Args<'_>, vm: &VirtualMachine) -> PyResult { - let mut fd2 = std::mem::ManuallyDrop::new(args.fd2); + let mut fd2 = core::mem::ManuallyDrop::new(args.fd2); nix::unistd::dup2(args.fd, &mut fd2).map_err(|e| e.into_pyexception(vm))?; - let fd2 = std::mem::ManuallyDrop::into_inner(fd2); + let fd2 = core::mem::ManuallyDrop::into_inner(fd2); if !args.inheritable { super::set_inheritable(fd2.as_fd(), false).map_err(|e| e.into_pyexception(vm))? } diff --git a/crates/vm/src/stdlib/pwd.rs b/crates/vm/src/stdlib/pwd.rs index e4d7075dbc8..6405ed7be91 100644 --- a/crates/vm/src/stdlib/pwd.rs +++ b/crates/vm/src/stdlib/pwd.rs @@ -37,7 +37,7 @@ mod pwd { impl From for PasswdData { fn from(user: User) -> Self { // this is just a pain... - let cstr_lossy = |s: std::ffi::CString| { + let cstr_lossy = |s: alloc::ffi::CString| { s.into_string() .unwrap_or_else(|e| e.into_cstring().to_string_lossy().into_owned()) }; @@ -105,7 +105,7 @@ mod pwd { let mut list = Vec::new(); unsafe { libc::setpwent() }; - while let Some(ptr) = std::ptr::NonNull::new(unsafe { libc::getpwent() }) { + while let Some(ptr) = core::ptr::NonNull::new(unsafe { libc::getpwent() }) { let user = User::from(unsafe { ptr.as_ref() }); let passwd = PasswdData::from(user).to_pyobject(vm); list.push(passwd); diff --git a/crates/vm/src/stdlib/signal.rs b/crates/vm/src/stdlib/signal.rs index 810ffabefe6..dd0d9a7a96f 100644 --- a/crates/vm/src/stdlib/signal.rs +++ b/crates/vm/src/stdlib/signal.rs @@ -24,7 +24,7 @@ pub(crate) mod _signal { builtins::PyTypeRef, function::{ArgIntoFloat, OptionalArg}, }; - use std::sync::atomic::{self, Ordering}; + use core::sync::atomic::{self, Ordering}; #[cfg(any(unix, windows))] use libc::sighandler_t; @@ -301,7 +301,7 @@ pub(crate) mod _signal { it_value: double_to_timeval(seconds), it_interval: double_to_timeval(interval), }; - let mut old = std::mem::MaybeUninit::::uninit(); + let mut old = core::mem::MaybeUninit::::uninit(); #[cfg(any(target_os = "linux", target_os = "android"))] let ret = unsafe { ffi::setitimer(which, &new, old.as_mut_ptr()) }; #[cfg(not(any(target_os = "linux", target_os = "android")))] @@ -318,7 +318,7 @@ pub(crate) mod _signal { #[cfg(unix)] #[pyfunction] fn getitimer(which: i32, vm: &VirtualMachine) -> PyResult<(f64, f64)> { - let mut old = std::mem::MaybeUninit::::uninit(); + let mut old = core::mem::MaybeUninit::::uninit(); #[cfg(any(target_os = "linux", target_os = "android"))] let ret = unsafe { ffi::getitimer(which, old.as_mut_ptr()) }; #[cfg(not(any(target_os = "linux", target_os = "android")))] @@ -489,7 +489,7 @@ pub(crate) mod _signal { if s.is_null() { Ok(None) } else { - let cstr = unsafe { std::ffi::CStr::from_ptr(s) }; + let cstr = unsafe { core::ffi::CStr::from_ptr(s) }; Ok(Some(cstr.to_string_lossy().into_owned())) } } @@ -522,7 +522,7 @@ pub(crate) mod _signal { #[cfg(unix)] { // Use sigfillset to get all valid signals - let mut mask: libc::sigset_t = unsafe { std::mem::zeroed() }; + let mut mask: libc::sigset_t = unsafe { core::mem::zeroed() }; // SAFETY: mask is a valid pointer if unsafe { libc::sigfillset(&mut mask) } != 0 { return Err(vm.new_os_error("sigfillset failed".to_owned())); @@ -580,7 +580,7 @@ pub(crate) mod _signal { use crate::convert::IntoPyException; // Initialize sigset - let mut sigset: libc::sigset_t = unsafe { std::mem::zeroed() }; + let mut sigset: libc::sigset_t = unsafe { core::mem::zeroed() }; // SAFETY: sigset is a valid pointer if unsafe { libc::sigemptyset(&mut sigset) } != 0 { return Err(std::io::Error::last_os_error().into_pyexception(vm)); @@ -611,7 +611,7 @@ pub(crate) mod _signal { } // Call pthread_sigmask - let mut old_mask: libc::sigset_t = unsafe { std::mem::zeroed() }; + let mut old_mask: libc::sigset_t = unsafe { core::mem::zeroed() }; // SAFETY: all pointers are valid let err = unsafe { libc::pthread_sigmask(how, &sigset, &mut old_mask) }; if err != 0 { diff --git a/crates/vm/src/stdlib/string.rs b/crates/vm/src/stdlib/string.rs index 576cae62775..a9911f3d45f 100644 --- a/crates/vm/src/stdlib/string.rs +++ b/crates/vm/src/stdlib/string.rs @@ -16,7 +16,7 @@ mod _string { convert::ToPyException, convert::ToPyObject, }; - use std::mem; + use core::mem; fn create_format_part( literal: Wtf8Buf, diff --git a/crates/vm/src/stdlib/symtable.rs b/crates/vm/src/stdlib/symtable.rs index 8a142857787..51c5c8e47ea 100644 --- a/crates/vm/src/stdlib/symtable.rs +++ b/crates/vm/src/stdlib/symtable.rs @@ -7,10 +7,10 @@ mod symtable { builtins::{PyDictRef, PyStrRef}, compiler, }; + use alloc::fmt; use rustpython_codegen::symboltable::{ CompilerScope, Symbol, SymbolFlags, SymbolScope, SymbolTable, }; - use std::fmt; // Consts as defined at // https://github.com/python/cpython/blob/6cb20a219a860eaf687b2d968b41c480c7461909/Include/internal/pycore_symtable.h#L156 @@ -180,7 +180,7 @@ mod symtable { #[pygetset] fn id(&self) -> usize { - self as *const Self as *const std::ffi::c_void as usize + self as *const Self as *const core::ffi::c_void as usize } #[pygetset] diff --git a/crates/vm/src/stdlib/sys.rs b/crates/vm/src/stdlib/sys.rs index 92b15878b46..30f126b3742 100644 --- a/crates/vm/src/stdlib/sys.rs +++ b/crates/vm/src/stdlib/sys.rs @@ -24,11 +24,11 @@ mod sys { version, vm::{Settings, VirtualMachine}, }; + use core::sync::atomic::Ordering; use num_traits::ToPrimitive; use std::{ env::{self, VarError}, io::Read, - sync::atomic::Ordering, }; #[cfg(windows)] @@ -67,7 +67,7 @@ mod sys { #[pyattr(name = "maxsize")] pub(crate) const MAXSIZE: isize = isize::MAX; #[pyattr(name = "maxunicode")] - const MAXUNICODE: u32 = std::char::MAX as u32; + const MAXUNICODE: u32 = core::char::MAX as u32; #[pyattr(name = "platform")] pub(crate) const PLATFORM: &str = { cfg_if::cfg_if! { @@ -751,7 +751,7 @@ mod sys { let sizeof = || -> PyResult { let res = vm.call_special_method(&args.obj, identifier!(vm, __sizeof__), ())?; let res = res.try_index(vm)?.try_to_primitive::(vm)?; - Ok(res + std::mem::size_of::()) + Ok(res + core::mem::size_of::()) }; sizeof() .map(|x| vm.ctx.new_int(x).into()) @@ -1377,7 +1377,7 @@ mod sys { const INFO: Self = { use rustpython_common::hash::*; Self { - width: std::mem::size_of::() * 8, + width: core::mem::size_of::() * 8, modulus: MODULUS, inf: INF, nan: NAN, @@ -1407,7 +1407,7 @@ mod sys { impl IntInfoData { const INFO: Self = Self { bits_per_digit: 30, //? - sizeof_digit: std::mem::size_of::(), + sizeof_digit: core::mem::size_of::(), default_max_str_digits: 4300, str_digits_check_threshold: 640, }; @@ -1525,7 +1525,7 @@ pub(crate) fn init_module(vm: &VirtualMachine, module: &Py, builtins: pub struct PyStderr<'vm>(pub &'vm VirtualMachine); impl PyStderr<'_> { - pub fn write_fmt(&self, args: std::fmt::Arguments<'_>) { + pub fn write_fmt(&self, args: core::fmt::Arguments<'_>) { use crate::py_io::Write; let vm = self.0; diff --git a/crates/vm/src/stdlib/thread.rs b/crates/vm/src/stdlib/thread.rs index fd300ac2f74..6ab754be094 100644 --- a/crates/vm/src/stdlib/thread.rs +++ b/crates/vm/src/stdlib/thread.rs @@ -11,12 +11,14 @@ pub(crate) mod _thread { function::{ArgCallable, Either, FuncArgs, KwArgs, OptionalArg, PySetterValue}, types::{Constructor, GetAttr, Representable, SetAttr}, }; + use alloc::fmt; + use core::{cell::RefCell, time::Duration}; use crossbeam_utils::atomic::AtomicCell; use parking_lot::{ RawMutex, RawThreadId, lock_api::{RawMutex as RawMutexT, RawMutexTimed, RawReentrantMutex}, }; - use std::{cell::RefCell, fmt, thread, time::Duration}; + use std::thread; use thread_local::ThreadLocal; // PYTHREAD_NAME: show current thread name @@ -151,7 +153,7 @@ pub(crate) mod _thread { let new_mut = RawMutex::INIT; unsafe { - let old_mutex: &AtomicCell = std::mem::transmute(&self.mu); + let old_mutex: &AtomicCell = core::mem::transmute(&self.mu); old_mutex.swap(new_mut); } @@ -287,7 +289,7 @@ pub(crate) mod _thread { } fn thread_to_id(t: &thread::Thread) -> u64 { - use std::hash::{Hash, Hasher}; + use core::hash::{Hash, Hasher}; struct U64Hash { v: Option, } diff --git a/crates/vm/src/stdlib/time.rs b/crates/vm/src/stdlib/time.rs index b9b53cdc5c5..97d60ae98a1 100644 --- a/crates/vm/src/stdlib/time.rs +++ b/crates/vm/src/stdlib/time.rs @@ -21,12 +21,12 @@ pub(crate) fn make_module(vm: &VirtualMachine) -> PyRef { unsafe extern "C" { #[cfg(not(target_os = "freebsd"))] #[link_name = "daylight"] - static c_daylight: std::ffi::c_int; + static c_daylight: core::ffi::c_int; // pub static dstbias: std::ffi::c_int; #[link_name = "timezone"] - static c_timezone: std::ffi::c_long; + static c_timezone: core::ffi::c_long; #[link_name = "tzname"] - static c_tzname: [*const std::ffi::c_char; 2]; + static c_tzname: [*const core::ffi::c_char; 2]; #[link_name = "tzset"] fn c_tzset(); } @@ -43,7 +43,7 @@ mod decl { DateTime, Datelike, TimeZone, Timelike, naive::{NaiveDate, NaiveDateTime, NaiveTime}, }; - use std::time::Duration; + use core::time::Duration; #[cfg(target_env = "msvc")] #[cfg(not(target_arch = "wasm32"))] use windows_sys::Win32::System::Time::{GetTimeZoneInformation, TIME_ZONE_INFORMATION}; @@ -104,7 +104,7 @@ mod decl { { // this is basically std::thread::sleep, but that catches interrupts and we don't want to; let ts = nix::sys::time::TimeSpec::from(dur); - let res = unsafe { libc::nanosleep(ts.as_ref(), std::ptr::null_mut()) }; + let res = unsafe { libc::nanosleep(ts.as_ref(), core::ptr::null_mut()) }; let interrupted = res == -1 && nix::Error::last_raw() == libc::EINTR; if interrupted { @@ -200,7 +200,7 @@ mod decl { #[cfg(not(target_env = "msvc"))] #[cfg(not(target_arch = "wasm32"))] #[pyattr] - fn timezone(_vm: &VirtualMachine) -> std::ffi::c_long { + fn timezone(_vm: &VirtualMachine) -> core::ffi::c_long { unsafe { super::c_timezone } } @@ -217,7 +217,7 @@ mod decl { #[cfg(not(target_env = "msvc"))] #[cfg(not(target_arch = "wasm32"))] #[pyattr] - fn daylight(_vm: &VirtualMachine) -> std::ffi::c_int { + fn daylight(_vm: &VirtualMachine) -> core::ffi::c_int { unsafe { super::c_daylight } } @@ -236,8 +236,8 @@ mod decl { fn tzname(vm: &VirtualMachine) -> crate::builtins::PyTupleRef { use crate::builtins::tuple::IntoPyTuple; - unsafe fn to_str(s: *const std::ffi::c_char) -> String { - unsafe { std::ffi::CStr::from_ptr(s) } + unsafe fn to_str(s: *const core::ffi::c_char) -> String { + unsafe { core::ffi::CStr::from_ptr(s) } .to_string_lossy() .into_owned() } @@ -357,7 +357,7 @@ mod decl { t: OptionalArg, vm: &VirtualMachine, ) -> PyResult { - use std::fmt::Write; + use core::fmt::Write; let instant = t.naive_or_local(vm)?; @@ -500,8 +500,8 @@ mod decl { pub tm_zone: PyObjectRef, } - impl std::fmt::Debug for StructTimeData { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + impl core::fmt::Debug for StructTimeData { + fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result { write!(f, "struct_time()") } } @@ -590,8 +590,8 @@ mod platform { builtins::{PyNamespace, PyStrRef}, convert::IntoPyException, }; + use core::time::Duration; use nix::{sys::time::TimeSpec, time::ClockId}; - use std::time::Duration; #[cfg(target_os = "solaris")] #[pyattr] @@ -818,7 +818,7 @@ mod platform { fn u64_from_filetime(time: FILETIME) -> u64 { let large: [u32; 2] = [time.dwLowDateTime, time.dwHighDateTime]; - unsafe { std::mem::transmute(large) } + unsafe { core::mem::transmute(large) } } fn win_perf_counter_frequency(vm: &VirtualMachine) -> PyResult { diff --git a/crates/vm/src/stdlib/winapi.rs b/crates/vm/src/stdlib/winapi.rs index a3aba447489..7279f9776e2 100644 --- a/crates/vm/src/stdlib/winapi.rs +++ b/crates/vm/src/stdlib/winapi.rs @@ -108,7 +108,7 @@ mod _winapi { WindowsSysResult(windows_sys::Win32::System::Pipes::CreatePipe( read.as_mut_ptr(), write.as_mut_ptr(), - std::ptr::null(), + core::ptr::null(), size, )) .to_pyresult(vm)?; @@ -279,8 +279,8 @@ mod _winapi { WindowsSysResult(windows_sys::Win32::System::Threading::CreateProcessW( app_name, command_line, - std::ptr::null(), - std::ptr::null(), + core::ptr::null(), + core::ptr::null(), args.inherit_handles, args.creation_flags | windows_sys::Win32::System::Threading::EXTENDED_STARTUPINFO_PRESENT @@ -455,7 +455,7 @@ mod _winapi { handlelist.as_mut_ptr() as _, (handlelist.len() * std::mem::size_of::()) as _, std::ptr::null_mut(), - std::ptr::null(), + core::ptr::null(), ) }) .into_pyresult(vm)?; @@ -874,7 +874,7 @@ mod _winapi { } let buf = buffer.borrow_buf(); - let len = std::cmp::min(buf.len(), u32::MAX as usize) as u32; + let len = core::cmp::min(buf.len(), u32::MAX as usize) as u32; let mut written: u32 = 0; let ret = unsafe { @@ -948,7 +948,7 @@ mod _winapi { let mut batches: Vec> = Vec::new(); let mut i = 0; while i < nhandles { - let end = std::cmp::min(i + batch_size, nhandles); + let end = core::cmp::min(i + batch_size, nhandles); batches.push(handles[i..end].to_vec()); i = end; } diff --git a/crates/vm/src/stdlib/winreg.rs b/crates/vm/src/stdlib/winreg.rs index b5e568fce6d..f3d8ca10768 100644 --- a/crates/vm/src/stdlib/winreg.rs +++ b/crates/vm/src/stdlib/winreg.rs @@ -367,10 +367,10 @@ mod winreg { key, wide_sub_key.as_ptr(), args.reserved, - std::ptr::null(), + core::ptr::null(), Registry::REG_OPTION_NON_VOLATILE, args.access, - std::ptr::null(), + core::ptr::null(), &mut res, std::ptr::null_mut(), ) @@ -404,7 +404,9 @@ mod winreg { #[pyfunction] fn DeleteValue(key: PyRef, value: Option, vm: &VirtualMachine) -> PyResult<()> { let wide_value = value.map(|v| v.to_wide_with_nul()); - let value_ptr = wide_value.as_ref().map_or(std::ptr::null(), |v| v.as_ptr()); + let value_ptr = wide_value + .as_ref() + .map_or(core::ptr::null(), |v| v.as_ptr()); let res = unsafe { Registry::RegDeleteValueW(key.hkey.load(), value_ptr) }; if res == 0 { Ok(()) @@ -713,7 +715,7 @@ mod winreg { let res = unsafe { Registry::RegQueryValueExW( target_key, - std::ptr::null(), // NULL value name for default value + core::ptr::null(), // NULL value name for default value std::ptr::null_mut(), &mut reg_type, buffer.as_mut_ptr(), @@ -871,10 +873,10 @@ mod winreg { hkey, wide_sub_key.as_ptr(), 0, - std::ptr::null(), + core::ptr::null(), 0, Registry::KEY_SET_VALUE, - std::ptr::null(), + core::ptr::null(), &mut out_key, std::ptr::null_mut(), ) @@ -893,7 +895,7 @@ mod winreg { let res = unsafe { Registry::RegSetValueExW( target_key, - std::ptr::null(), // value name is NULL + core::ptr::null(), // value name is NULL 0, typ, wide_value.as_ptr() as *const u8, @@ -1104,7 +1106,7 @@ mod winreg { } Ok(None) => { let len = 0; - let ptr = std::ptr::null(); + let ptr = core::ptr::null(); let wide_value_name = value_name.to_wide_with_nul(); let res = unsafe { Registry::RegSetValueExW( diff --git a/crates/vm/src/suggestion.rs b/crates/vm/src/suggestion.rs index 866deb668eb..55326d1d3f0 100644 --- a/crates/vm/src/suggestion.rs +++ b/crates/vm/src/suggestion.rs @@ -7,8 +7,8 @@ use crate::{ exceptions::types::PyBaseException, sliceable::SliceableSequenceOp, }; +use core::iter::ExactSizeIterator; use rustpython_common::str::levenshtein::{MOVE_COST, levenshtein_distance}; -use std::iter::ExactSizeIterator; const MAX_CANDIDATE_ITEMS: usize = 750; diff --git a/crates/vm/src/types/slot.rs b/crates/vm/src/types/slot.rs index 658e21cba8a..9d7d09f2d18 100644 --- a/crates/vm/src/types/slot.rs +++ b/crates/vm/src/types/slot.rs @@ -17,9 +17,9 @@ use crate::{ types::slot_defs::{SlotAccessor, find_slot_defs_by_name}, vm::Context, }; +use core::{any::Any, any::TypeId, borrow::Borrow, cmp::Ordering, ops::Deref}; use crossbeam_utils::atomic::AtomicCell; use num_traits::{Signed, ToPrimitive}; -use std::{any::Any, any::TypeId, borrow::Borrow, cmp::Ordering, ops::Deref}; /// Type-erased storage for extension module data attached to heap types. pub struct TypeDataSlot { @@ -71,7 +71,7 @@ impl<'a, T: Any + 'static> TypeDataRef<'a, T> { } } -impl std::ops::Deref for TypeDataRef<'_, T> { +impl core::ops::Deref for TypeDataRef<'_, T> { type Target = T; fn deref(&self) -> &Self::Target { @@ -96,7 +96,7 @@ impl<'a, T: Any + 'static> TypeDataRefMut<'a, T> { } } -impl std::ops::Deref for TypeDataRefMut<'_, T> { +impl core::ops::Deref for TypeDataRefMut<'_, T> { type Target = T; fn deref(&self) -> &Self::Target { @@ -104,7 +104,7 @@ impl std::ops::Deref for TypeDataRefMut<'_, T> { } } -impl std::ops::DerefMut for TypeDataRefMut<'_, T> { +impl core::ops::DerefMut for TypeDataRefMut<'_, T> { fn deref_mut(&mut self) -> &mut Self::Target { &mut self.guard } @@ -203,8 +203,8 @@ impl PyTypeSlots { } } -impl std::fmt::Debug for PyTypeSlots { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { +impl core::fmt::Debug for PyTypeSlots { + fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result { f.write_str("PyTypeSlots") } } @@ -1310,7 +1310,7 @@ impl PyType { /// - Special class type handling (e.g., `PyType` and its metaclasses) /// - Post-creation mutations that require `PyRef` #[pyclass] -pub trait Constructor: PyPayload + std::fmt::Debug { +pub trait Constructor: PyPayload + core::fmt::Debug { type Args: FromArgs; /// The type slot for `__new__`. Override this only when you need special @@ -1328,7 +1328,7 @@ pub trait Constructor: PyPayload + std::fmt::Debug { fn py_new(cls: &Py, args: Self::Args, vm: &VirtualMachine) -> PyResult; } -pub trait DefaultConstructor: PyPayload + Default + std::fmt::Debug { +pub trait DefaultConstructor: PyPayload + Default + core::fmt::Debug { fn construct_and_init(args: Self::Args, vm: &VirtualMachine) -> PyResult> where Self: Initializer, @@ -1862,7 +1862,7 @@ pub trait Iterable: PyPayload { fn extend_slots(_slots: &mut PyTypeSlots) {} } -// `Iterator` fits better, but to avoid confusion with rust std::iter::Iterator +// `Iterator` fits better, but to avoid confusion with rust core::iter::Iterator #[pyclass(with(Iterable))] pub trait IterNext: PyPayload + Iterable { #[pyslot] diff --git a/crates/vm/src/utils.rs b/crates/vm/src/utils.rs index af34405c7be..db232e81949 100644 --- a/crates/vm/src/utils.rs +++ b/crates/vm/src/utils.rs @@ -14,15 +14,15 @@ pub fn hash_iter<'a, I: IntoIterator>( vm.state.hash_secret.hash_iter(iter, |obj| obj.hash(vm)) } -impl ToPyObject for std::convert::Infallible { +impl ToPyObject for core::convert::Infallible { fn to_pyobject(self, _vm: &VirtualMachine) -> PyObjectRef { match self {} } } pub trait ToCString: AsRef { - fn to_cstring(&self, vm: &VirtualMachine) -> PyResult { - std::ffi::CString::new(self.as_ref().as_bytes()).map_err(|err| err.to_pyexception(vm)) + fn to_cstring(&self, vm: &VirtualMachine) -> PyResult { + alloc::ffi::CString::new(self.as_ref().as_bytes()).map_err(|err| err.to_pyexception(vm)) } fn ensure_no_nul(&self, vm: &VirtualMachine) -> PyResult<()> { if self.as_ref().as_bytes().contains(&b'\0') { @@ -45,7 +45,7 @@ pub(crate) fn collection_repr<'a, I>( vm: &VirtualMachine, ) -> PyResult where - I: std::iter::Iterator, + I: core::iter::Iterator, { let mut repr = String::new(); if let Some(name) = class_name { diff --git a/crates/vm/src/version.rs b/crates/vm/src/version.rs index 0a598842a56..cc118ee5b0e 100644 --- a/crates/vm/src/version.rs +++ b/crates/vm/src/version.rs @@ -1,7 +1,8 @@ //! Several function to retrieve version information. use chrono::{Local, prelude::DateTime}; -use std::time::{Duration, UNIX_EPOCH}; +use core::time::Duration; +use std::time::UNIX_EPOCH; // = 3.13.0alpha pub const MAJOR: usize = 3; diff --git a/crates/vm/src/vm/context.rs b/crates/vm/src/vm/context.rs index 486c1861fb1..b12352f6eee 100644 --- a/crates/vm/src/vm/context.rs +++ b/crates/vm/src/vm/context.rs @@ -264,7 +264,7 @@ declare_const_name! { // Basic objects: impl Context { - pub const INT_CACHE_POOL_RANGE: std::ops::RangeInclusive = (-5)..=256; + pub const INT_CACHE_POOL_RANGE: core::ops::RangeInclusive = (-5)..=256; const INT_CACHE_POOL_MIN: i32 = *Self::INT_CACHE_POOL_RANGE.start(); pub fn genesis() -> &'static PyRc { @@ -374,14 +374,14 @@ impl Context { #[inline] pub fn empty_tuple_typed(&self) -> &Py> { let py: &Py = &self.empty_tuple; - unsafe { std::mem::transmute(py) } + unsafe { core::mem::transmute(py) } } // universal pyref constructor pub fn new_pyref(&self, value: T) -> PyRef

where T: Into

, - P: PyPayload + std::fmt::Debug, + P: PyPayload + core::fmt::Debug, { value.into().into_ref(self) } diff --git a/crates/vm/src/vm/interpreter.rs b/crates/vm/src/vm/interpreter.rs index 503feb3dc7f..8d37ad6c840 100644 --- a/crates/vm/src/vm/interpreter.rs +++ b/crates/vm/src/vm/interpreter.rs @@ -1,6 +1,6 @@ use super::{Context, PyConfig, VirtualMachine, setting::Settings, thread}; use crate::{PyResult, getpath, stdlib::atexit, vm::PyBaseExceptionRef}; -use std::sync::atomic::Ordering; +use core::sync::atomic::Ordering; /// The general interface for the VM /// diff --git a/crates/vm/src/vm/mod.rs b/crates/vm/src/vm/mod.rs index 34092454059..7c527b3e0da 100644 --- a/crates/vm/src/vm/mod.rs +++ b/crates/vm/src/vm/mod.rs @@ -33,6 +33,11 @@ use crate::{ signal, stdlib, warn::WarningsState, }; +use alloc::borrow::Cow; +use core::{ + cell::{Cell, Ref, RefCell}, + sync::atomic::AtomicBool, +}; use crossbeam_utils::atomic::AtomicCell; #[cfg(unix)] use nix::{ @@ -40,11 +45,8 @@ use nix::{ unistd::getpid, }; use std::{ - borrow::Cow, - cell::{Cell, Ref, RefCell}, collections::{HashMap, HashSet}, ffi::{OsStr, OsString}, - sync::atomic::AtomicBool, }; pub use context::Context; @@ -789,14 +791,14 @@ impl VirtualMachine { pub(crate) fn push_exception(&self, exc: Option) { let mut excs = self.exceptions.borrow_mut(); - let prev = std::mem::take(&mut *excs); + let prev = core::mem::take(&mut *excs); excs.prev = Some(Box::new(prev)); excs.exc = exc } pub(crate) fn pop_exception(&self) -> Option { let mut excs = self.exceptions.borrow_mut(); - let cur = std::mem::take(&mut *excs); + let cur = core::mem::take(&mut *excs); *excs = *cur.prev.expect("pop_exception() without nested exc stack"); cur.exc } @@ -811,7 +813,7 @@ impl VirtualMachine { pub(crate) fn set_exception(&self, exc: Option) { // don't be holding the RefCell guard while __del__ is called - let prev = std::mem::replace(&mut self.exceptions.borrow_mut().exc, exc); + let prev = core::mem::replace(&mut self.exceptions.borrow_mut().exc, exc); drop(prev); } @@ -984,7 +986,7 @@ impl AsRef for VirtualMachine { } fn core_frozen_inits() -> impl Iterator { - let iter = std::iter::empty(); + let iter = core::iter::empty(); macro_rules! ext_modules { ($iter:ident, $($t:tt)*) => { let $iter = $iter.chain(py_freeze!($($t)*)); diff --git a/crates/vm/src/vm/thread.rs b/crates/vm/src/vm/thread.rs index 2e687d99820..7e8f0f87e56 100644 --- a/crates/vm/src/vm/thread.rs +++ b/crates/vm/src/vm/thread.rs @@ -1,10 +1,10 @@ use crate::{AsObject, PyObject, PyObjectRef, VirtualMachine}; -use itertools::Itertools; -use std::{ +use core::{ cell::{Cell, RefCell}, ptr::NonNull, - thread_local, }; +use itertools::Itertools; +use std::thread_local; thread_local! { pub(super) static VM_STACK: RefCell>> = Vec::with_capacity(1).into(); diff --git a/crates/vm/src/vm/vm_new.rs b/crates/vm/src/vm/vm_new.rs index 6d0e983c844..ba09c8ecf69 100644 --- a/crates/vm/src/vm/vm_new.rs +++ b/crates/vm/src/vm/vm_new.rs @@ -95,7 +95,7 @@ impl VirtualMachine { pub fn new_exception(&self, exc_type: PyTypeRef, args: Vec) -> PyBaseExceptionRef { debug_assert_eq!( exc_type.slots.basicsize, - std::mem::size_of::(), + core::mem::size_of::(), "vm.new_exception() is only for exception types without additional payload. The given type '{}' is not allowed.", exc_type.class().name() ); @@ -118,7 +118,7 @@ impl VirtualMachine { errno: Option, msg: impl ToPyObject, ) -> PyRef { - debug_assert_eq!(exc_type.slots.basicsize, std::mem::size_of::()); + debug_assert_eq!(exc_type.slots.basicsize, core::mem::size_of::()); OSErrorBuilder::with_subtype(exc_type, errno, msg, self).build(self) } diff --git a/crates/vm/src/vm/vm_ops.rs b/crates/vm/src/vm/vm_ops.rs index 635fa10e630..1d466984377 100644 --- a/crates/vm/src/vm/vm_ops.rs +++ b/crates/vm/src/vm/vm_ops.rs @@ -302,8 +302,8 @@ impl VirtualMachine { } if let Some(slot_c) = class_c.slots.as_number.left_ternary_op(op_slot) - && slot_a.is_some_and(|slot_a| !std::ptr::fn_addr_eq(slot_a, slot_c)) - && slot_b.is_some_and(|slot_b| !std::ptr::fn_addr_eq(slot_b, slot_c)) + && slot_a.is_some_and(|slot_a| !core::ptr::fn_addr_eq(slot_a, slot_c)) + && slot_b.is_some_and(|slot_b| !core::ptr::fn_addr_eq(slot_b, slot_c)) { let ret = slot_c(a, b, c, self)?; if !ret.is(&self.ctx.not_implemented) { diff --git a/crates/vm/src/warn.rs b/crates/vm/src/warn.rs index 3dbd43ab537..09d48078e56 100644 --- a/crates/vm/src/warn.rs +++ b/crates/vm/src/warn.rs @@ -60,7 +60,7 @@ fn get_warnings_attr( && !vm .state .finalizing - .load(std::sync::atomic::Ordering::SeqCst) + .load(core::sync::atomic::Ordering::SeqCst) { match vm.import("warnings", 0) { Ok(module) => module, diff --git a/crates/vm/src/windows.rs b/crates/vm/src/windows.rs index ccf940811b8..ff2b612c06d 100644 --- a/crates/vm/src/windows.rs +++ b/crates/vm/src/windows.rs @@ -301,7 +301,7 @@ fn win32_xstat_slow_impl(path: &OsStr, traverse: bool) -> std::io::Result std::io::Result std::io::Result PyResult<()> { } #[cfg(feature = "flame-it")] -fn write_profile(settings: &Settings) -> Result<(), Box> { +fn write_profile(settings: &Settings) -> Result<(), Box> { use std::{fs, io}; enum ProfileFormat { From 5e1fc93f5043c1f054122e7220416b4979521dee Mon Sep 17 00:00:00 2001 From: Jack O'Connor Date: Tue, 30 Dec 2025 04:42:47 +0000 Subject: [PATCH 0002/1575] Update zipfile to 3.13.5 (#6069) * Add Lib/test/archiver_tests.py @ 3.13.5 Needed for updated zipfile tests. * Update zipfile to 3.13.5 Notes: - I have to skip some brand new tests due to shift_jis encoding not being supported - `test_write_filtered_python_package` marked as `expectedFailure` with "AttributeError: module 'os' has no attribute 'supports_effective_ids'" - I didn't want to do a partial or full update to os module in this PR --------- Co-authored-by: Jack O'Connor --- Lib/test/archiver_tests.py | 177 +++ Lib/test/test_zipfile/__init__.py | 5 + Lib/test/test_zipfile/__main__.py | 7 + Lib/test/test_zipfile/_path/__init__.py | 0 Lib/test/test_zipfile/_path/_functools.py | 9 + Lib/test/test_zipfile/_path/_itertools.py | 79 + Lib/test/test_zipfile/_path/_support.py | 9 + Lib/test/test_zipfile/_path/_test_params.py | 39 + .../test_zipfile/_path/test_complexity.py | 105 ++ Lib/test/test_zipfile/_path/test_path.py | 687 +++++++++ Lib/test/test_zipfile/_path/write-alpharep.py | 3 + .../test_core.py} | 1291 ++++++++++++----- Lib/{zipfile.py => zipfile/__init__.py} | 737 ++++------ Lib/zipfile/__main__.py | 4 + Lib/zipfile/_path/__init__.py | 452 ++++++ Lib/zipfile/_path/glob.py | 113 ++ 16 files changed, 2890 insertions(+), 827 deletions(-) create mode 100644 Lib/test/archiver_tests.py create mode 100644 Lib/test/test_zipfile/__init__.py create mode 100644 Lib/test/test_zipfile/__main__.py create mode 100644 Lib/test/test_zipfile/_path/__init__.py create mode 100644 Lib/test/test_zipfile/_path/_functools.py create mode 100644 Lib/test/test_zipfile/_path/_itertools.py create mode 100644 Lib/test/test_zipfile/_path/_support.py create mode 100644 Lib/test/test_zipfile/_path/_test_params.py create mode 100644 Lib/test/test_zipfile/_path/test_complexity.py create mode 100644 Lib/test/test_zipfile/_path/test_path.py create mode 100644 Lib/test/test_zipfile/_path/write-alpharep.py rename Lib/test/{test_zipfile.py => test_zipfile/test_core.py} (72%) rename Lib/{zipfile.py => zipfile/__init__.py} (85%) create mode 100644 Lib/zipfile/__main__.py create mode 100644 Lib/zipfile/_path/__init__.py create mode 100644 Lib/zipfile/_path/glob.py diff --git a/Lib/test/archiver_tests.py b/Lib/test/archiver_tests.py new file mode 100644 index 00000000000..24745941b08 --- /dev/null +++ b/Lib/test/archiver_tests.py @@ -0,0 +1,177 @@ +"""Tests common to tarfile and zipfile.""" + +import os +import sys + +from test.support import swap_attr +from test.support import os_helper + +class OverwriteTests: + + def setUp(self): + os.makedirs(self.testdir) + self.addCleanup(os_helper.rmtree, self.testdir) + + def create_file(self, path, content=b''): + with open(path, 'wb') as f: + f.write(content) + + def open(self, path): + raise NotImplementedError + + def extractall(self, ar): + raise NotImplementedError + + + def test_overwrite_file_as_file(self): + target = os.path.join(self.testdir, 'test') + self.create_file(target, b'content') + with self.open(self.ar_with_file) as ar: + self.extractall(ar) + self.assertTrue(os.path.isfile(target)) + with open(target, 'rb') as f: + self.assertEqual(f.read(), b'newcontent') + + def test_overwrite_dir_as_dir(self): + target = os.path.join(self.testdir, 'test') + os.mkdir(target) + with self.open(self.ar_with_dir) as ar: + self.extractall(ar) + self.assertTrue(os.path.isdir(target)) + + def test_overwrite_dir_as_implicit_dir(self): + target = os.path.join(self.testdir, 'test') + os.mkdir(target) + with self.open(self.ar_with_implicit_dir) as ar: + self.extractall(ar) + self.assertTrue(os.path.isdir(target)) + self.assertTrue(os.path.isfile(os.path.join(target, 'file'))) + with open(os.path.join(target, 'file'), 'rb') as f: + self.assertEqual(f.read(), b'newcontent') + + def test_overwrite_dir_as_file(self): + target = os.path.join(self.testdir, 'test') + os.mkdir(target) + with self.open(self.ar_with_file) as ar: + with self.assertRaises(PermissionError if sys.platform == 'win32' + else IsADirectoryError): + self.extractall(ar) + self.assertTrue(os.path.isdir(target)) + + def test_overwrite_file_as_dir(self): + target = os.path.join(self.testdir, 'test') + self.create_file(target, b'content') + with self.open(self.ar_with_dir) as ar: + with self.assertRaises(FileExistsError): + self.extractall(ar) + self.assertTrue(os.path.isfile(target)) + with open(target, 'rb') as f: + self.assertEqual(f.read(), b'content') + + def test_overwrite_file_as_implicit_dir(self): + target = os.path.join(self.testdir, 'test') + self.create_file(target, b'content') + with self.open(self.ar_with_implicit_dir) as ar: + with self.assertRaises(FileNotFoundError if sys.platform == 'win32' + else NotADirectoryError): + self.extractall(ar) + self.assertTrue(os.path.isfile(target)) + with open(target, 'rb') as f: + self.assertEqual(f.read(), b'content') + + @os_helper.skip_unless_symlink + def test_overwrite_file_symlink_as_file(self): + # XXX: It is potential security vulnerability. + target = os.path.join(self.testdir, 'test') + target2 = os.path.join(self.testdir, 'test2') + self.create_file(target2, b'content') + os.symlink('test2', target) + with self.open(self.ar_with_file) as ar: + self.extractall(ar) + self.assertTrue(os.path.islink(target)) + self.assertTrue(os.path.isfile(target2)) + with open(target2, 'rb') as f: + self.assertEqual(f.read(), b'newcontent') + + @os_helper.skip_unless_symlink + def test_overwrite_broken_file_symlink_as_file(self): + # XXX: It is potential security vulnerability. + target = os.path.join(self.testdir, 'test') + target2 = os.path.join(self.testdir, 'test2') + os.symlink('test2', target) + with self.open(self.ar_with_file) as ar: + self.extractall(ar) + self.assertTrue(os.path.islink(target)) + self.assertTrue(os.path.isfile(target2)) + with open(target2, 'rb') as f: + self.assertEqual(f.read(), b'newcontent') + + @os_helper.skip_unless_symlink + def test_overwrite_dir_symlink_as_dir(self): + # XXX: It is potential security vulnerability. + target = os.path.join(self.testdir, 'test') + target2 = os.path.join(self.testdir, 'test2') + os.mkdir(target2) + os.symlink('test2', target, target_is_directory=True) + with self.open(self.ar_with_dir) as ar: + self.extractall(ar) + self.assertTrue(os.path.islink(target)) + self.assertTrue(os.path.isdir(target2)) + + @os_helper.skip_unless_symlink + def test_overwrite_dir_symlink_as_implicit_dir(self): + # XXX: It is potential security vulnerability. + target = os.path.join(self.testdir, 'test') + target2 = os.path.join(self.testdir, 'test2') + os.mkdir(target2) + os.symlink('test2', target, target_is_directory=True) + with self.open(self.ar_with_implicit_dir) as ar: + self.extractall(ar) + self.assertTrue(os.path.islink(target)) + self.assertTrue(os.path.isdir(target2)) + self.assertTrue(os.path.isfile(os.path.join(target2, 'file'))) + with open(os.path.join(target2, 'file'), 'rb') as f: + self.assertEqual(f.read(), b'newcontent') + + @os_helper.skip_unless_symlink + def test_overwrite_broken_dir_symlink_as_dir(self): + target = os.path.join(self.testdir, 'test') + target2 = os.path.join(self.testdir, 'test2') + os.symlink('test2', target, target_is_directory=True) + with self.open(self.ar_with_dir) as ar: + with self.assertRaises(FileExistsError): + self.extractall(ar) + self.assertTrue(os.path.islink(target)) + self.assertFalse(os.path.exists(target2)) + + @os_helper.skip_unless_symlink + def test_overwrite_broken_dir_symlink_as_implicit_dir(self): + target = os.path.join(self.testdir, 'test') + target2 = os.path.join(self.testdir, 'test2') + os.symlink('test2', target, target_is_directory=True) + with self.open(self.ar_with_implicit_dir) as ar: + with self.assertRaises(FileExistsError): + self.extractall(ar) + self.assertTrue(os.path.islink(target)) + self.assertFalse(os.path.exists(target2)) + + def test_concurrent_extract_dir(self): + target = os.path.join(self.testdir, 'test') + def concurrent_mkdir(*args, **kwargs): + orig_mkdir(*args, **kwargs) + orig_mkdir(*args, **kwargs) + with swap_attr(os, 'mkdir', concurrent_mkdir) as orig_mkdir: + with self.open(self.ar_with_dir) as ar: + self.extractall(ar) + self.assertTrue(os.path.isdir(target)) + + def test_concurrent_extract_implicit_dir(self): + target = os.path.join(self.testdir, 'test') + def concurrent_mkdir(*args, **kwargs): + orig_mkdir(*args, **kwargs) + orig_mkdir(*args, **kwargs) + with swap_attr(os, 'mkdir', concurrent_mkdir) as orig_mkdir: + with self.open(self.ar_with_implicit_dir) as ar: + self.extractall(ar) + self.assertTrue(os.path.isdir(target)) + self.assertTrue(os.path.isfile(os.path.join(target, 'file'))) diff --git a/Lib/test/test_zipfile/__init__.py b/Lib/test/test_zipfile/__init__.py new file mode 100644 index 00000000000..4b16ecc3115 --- /dev/null +++ b/Lib/test/test_zipfile/__init__.py @@ -0,0 +1,5 @@ +import os +from test.support import load_package_tests + +def load_tests(*args): + return load_package_tests(os.path.dirname(__file__), *args) diff --git a/Lib/test/test_zipfile/__main__.py b/Lib/test/test_zipfile/__main__.py new file mode 100644 index 00000000000..e25ac946edf --- /dev/null +++ b/Lib/test/test_zipfile/__main__.py @@ -0,0 +1,7 @@ +import unittest + +from . import load_tests # noqa: F401 + + +if __name__ == "__main__": + unittest.main() diff --git a/Lib/test/test_zipfile/_path/__init__.py b/Lib/test/test_zipfile/_path/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/Lib/test/test_zipfile/_path/_functools.py b/Lib/test/test_zipfile/_path/_functools.py new file mode 100644 index 00000000000..75f2b20e06d --- /dev/null +++ b/Lib/test/test_zipfile/_path/_functools.py @@ -0,0 +1,9 @@ +import functools + + +# from jaraco.functools 3.5.2 +def compose(*funcs): + def compose_two(f1, f2): + return lambda *args, **kwargs: f1(f2(*args, **kwargs)) + + return functools.reduce(compose_two, funcs) diff --git a/Lib/test/test_zipfile/_path/_itertools.py b/Lib/test/test_zipfile/_path/_itertools.py new file mode 100644 index 00000000000..f735dd21733 --- /dev/null +++ b/Lib/test/test_zipfile/_path/_itertools.py @@ -0,0 +1,79 @@ +import itertools +from collections import deque +from itertools import islice + + +# from jaraco.itertools 6.3.0 +class Counter: + """ + Wrap an iterable in an object that stores the count of items + that pass through it. + + >>> items = Counter(range(20)) + >>> items.count + 0 + >>> values = list(items) + >>> items.count + 20 + """ + + def __init__(self, i): + self.count = 0 + self.iter = zip(itertools.count(1), i) + + def __iter__(self): + return self + + def __next__(self): + self.count, result = next(self.iter) + return result + + +# from more_itertools v8.13.0 +def always_iterable(obj, base_type=(str, bytes)): + if obj is None: + return iter(()) + + if (base_type is not None) and isinstance(obj, base_type): + return iter((obj,)) + + try: + return iter(obj) + except TypeError: + return iter((obj,)) + + +# from more_itertools v9.0.0 +def consume(iterator, n=None): + """Advance *iterable* by *n* steps. If *n* is ``None``, consume it + entirely. + Efficiently exhausts an iterator without returning values. Defaults to + consuming the whole iterator, but an optional second argument may be + provided to limit consumption. + >>> i = (x for x in range(10)) + >>> next(i) + 0 + >>> consume(i, 3) + >>> next(i) + 4 + >>> consume(i) + >>> next(i) + Traceback (most recent call last): + File "", line 1, in + StopIteration + If the iterator has fewer items remaining than the provided limit, the + whole iterator will be consumed. + >>> i = (x for x in range(3)) + >>> consume(i, 5) + >>> next(i) + Traceback (most recent call last): + File "", line 1, in + StopIteration + """ + # Use functions that consume iterators at C speed. + if n is None: + # feed the entire iterator into a zero-length deque + deque(iterator, maxlen=0) + else: + # advance to the empty slice starting at position n + next(islice(iterator, n, n), None) diff --git a/Lib/test/test_zipfile/_path/_support.py b/Lib/test/test_zipfile/_path/_support.py new file mode 100644 index 00000000000..1afdf3b3a77 --- /dev/null +++ b/Lib/test/test_zipfile/_path/_support.py @@ -0,0 +1,9 @@ +import importlib +import unittest + + +def import_or_skip(name): + try: + return importlib.import_module(name) + except ImportError: # pragma: no cover + raise unittest.SkipTest(f'Unable to import {name}') diff --git a/Lib/test/test_zipfile/_path/_test_params.py b/Lib/test/test_zipfile/_path/_test_params.py new file mode 100644 index 00000000000..00a9eaf2f99 --- /dev/null +++ b/Lib/test/test_zipfile/_path/_test_params.py @@ -0,0 +1,39 @@ +import functools +import types + +from ._itertools import always_iterable + + +def parameterize(names, value_groups): + """ + Decorate a test method to run it as a set of subtests. + + Modeled after pytest.parametrize. + """ + + def decorator(func): + @functools.wraps(func) + def wrapped(self): + for values in value_groups: + resolved = map(Invoked.eval, always_iterable(values)) + params = dict(zip(always_iterable(names), resolved)) + with self.subTest(**params): + func(self, **params) + + return wrapped + + return decorator + + +class Invoked(types.SimpleNamespace): + """ + Wrap a function to be invoked for each usage. + """ + + @classmethod + def wrap(cls, func): + return cls(func=func) + + @classmethod + def eval(cls, cand): + return cand.func() if isinstance(cand, cls) else cand diff --git a/Lib/test/test_zipfile/_path/test_complexity.py b/Lib/test/test_zipfile/_path/test_complexity.py new file mode 100644 index 00000000000..7c108fc6ab8 --- /dev/null +++ b/Lib/test/test_zipfile/_path/test_complexity.py @@ -0,0 +1,105 @@ +import io +import itertools +import math +import re +import string +import unittest +import zipfile + +from ._functools import compose +from ._itertools import consume +from ._support import import_or_skip + +big_o = import_or_skip('big_o') +pytest = import_or_skip('pytest') + + +class TestComplexity(unittest.TestCase): + @pytest.mark.flaky + def test_implied_dirs_performance(self): + best, others = big_o.big_o( + compose(consume, zipfile._path.CompleteDirs._implied_dirs), + lambda size: [ + '/'.join(string.ascii_lowercase + str(n)) for n in range(size) + ], + max_n=1000, + min_n=1, + ) + assert best <= big_o.complexities.Linear + + def make_zip_path(self, depth=1, width=1) -> zipfile.Path: + """ + Construct a Path with width files at every level of depth. + """ + zf = zipfile.ZipFile(io.BytesIO(), mode='w') + pairs = itertools.product(self.make_deep_paths(depth), self.make_names(width)) + for path, name in pairs: + zf.writestr(f"{path}{name}.txt", b'') + zf.filename = "big un.zip" + return zipfile.Path(zf) + + @classmethod + def make_names(cls, width, letters=string.ascii_lowercase): + """ + >>> list(TestComplexity.make_names(1)) + ['a'] + >>> list(TestComplexity.make_names(2)) + ['a', 'b'] + >>> list(TestComplexity.make_names(30)) + ['aa', 'ab', ..., 'bd'] + >>> list(TestComplexity.make_names(17124)) + ['aaa', 'aab', ..., 'zip'] + """ + # determine how many products are needed to produce width + n_products = max(1, math.ceil(math.log(width, len(letters)))) + inputs = (letters,) * n_products + combinations = itertools.product(*inputs) + names = map(''.join, combinations) + return itertools.islice(names, width) + + @classmethod + def make_deep_paths(cls, depth): + return map(cls.make_deep_path, range(depth)) + + @classmethod + def make_deep_path(cls, depth): + return ''.join(('d/',) * depth) + + def test_baseline_regex_complexity(self): + best, others = big_o.big_o( + lambda path: re.fullmatch(r'[^/]*\\.txt', path), + self.make_deep_path, + max_n=100, + min_n=1, + ) + assert best <= big_o.complexities.Constant + + @pytest.mark.flaky + def test_glob_depth(self): + best, others = big_o.big_o( + lambda path: consume(path.glob('*.txt')), + self.make_zip_path, + max_n=100, + min_n=1, + ) + assert best <= big_o.complexities.Linear + + @pytest.mark.flaky + def test_glob_width(self): + best, others = big_o.big_o( + lambda path: consume(path.glob('*.txt')), + lambda size: self.make_zip_path(width=size), + max_n=100, + min_n=1, + ) + assert best <= big_o.complexities.Linear + + @pytest.mark.flaky + def test_glob_width_and_depth(self): + best, others = big_o.big_o( + lambda path: consume(path.glob('*.txt')), + lambda size: self.make_zip_path(depth=size, width=size), + max_n=10, + min_n=1, + ) + assert best <= big_o.complexities.Linear diff --git a/Lib/test/test_zipfile/_path/test_path.py b/Lib/test/test_zipfile/_path/test_path.py new file mode 100644 index 00000000000..2d1c06cd968 --- /dev/null +++ b/Lib/test/test_zipfile/_path/test_path.py @@ -0,0 +1,687 @@ +import contextlib +import io +import itertools +import pathlib +import pickle +import stat +import sys +import time +import unittest +import zipfile +import zipfile._path + +from test.support.os_helper import FakePath, temp_dir + +from ._functools import compose +from ._itertools import Counter +from ._test_params import Invoked, parameterize + + +class jaraco: + class itertools: + Counter = Counter + + +def _make_link(info: zipfile.ZipInfo): # type: ignore[name-defined] + info.external_attr |= stat.S_IFLNK << 16 + + +def build_alpharep_fixture(): + """ + Create a zip file with this structure: + + . + ├── a.txt + ├── n.txt (-> a.txt) + ├── b + │ ├── c.txt + │ ├── d + │ │ └── e.txt + │ └── f.txt + ├── g + │ └── h + │ └── i.txt + └── j + ├── k.bin + ├── l.baz + └── m.bar + + This fixture has the following key characteristics: + + - a file at the root (a) + - a file two levels deep (b/d/e) + - multiple files in a directory (b/c, b/f) + - a directory containing only a directory (g/h) + - a directory with files of different extensions (j/klm) + - a symlink (n) pointing to (a) + + "alpha" because it uses alphabet + "rep" because it's a representative example + """ + data = io.BytesIO() + zf = zipfile.ZipFile(data, "w") + zf.writestr("a.txt", b"content of a") + zf.writestr("b/c.txt", b"content of c") + zf.writestr("b/d/e.txt", b"content of e") + zf.writestr("b/f.txt", b"content of f") + zf.writestr("g/h/i.txt", b"content of i") + zf.writestr("j/k.bin", b"content of k") + zf.writestr("j/l.baz", b"content of l") + zf.writestr("j/m.bar", b"content of m") + zf.writestr("n.txt", b"a.txt") + _make_link(zf.infolist()[-1]) + + zf.filename = "alpharep.zip" + return zf + + +alpharep_generators = [ + Invoked.wrap(build_alpharep_fixture), + Invoked.wrap(compose(zipfile._path.CompleteDirs.inject, build_alpharep_fixture)), +] + +pass_alpharep = parameterize(['alpharep'], alpharep_generators) + + +class TestPath(unittest.TestCase): + def setUp(self): + self.fixtures = contextlib.ExitStack() + self.addCleanup(self.fixtures.close) + + def zipfile_ondisk(self, alpharep): + tmpdir = pathlib.Path(self.fixtures.enter_context(temp_dir())) + buffer = alpharep.fp + alpharep.close() + path = tmpdir / alpharep.filename + with path.open("wb") as strm: + strm.write(buffer.getvalue()) + return path + + @pass_alpharep + def test_iterdir_and_types(self, alpharep): + root = zipfile.Path(alpharep) + assert root.is_dir() + a, n, b, g, j = root.iterdir() + assert a.is_file() + assert b.is_dir() + assert g.is_dir() + c, f, d = b.iterdir() + assert c.is_file() and f.is_file() + (e,) = d.iterdir() + assert e.is_file() + (h,) = g.iterdir() + (i,) = h.iterdir() + assert i.is_file() + + @pass_alpharep + def test_is_file_missing(self, alpharep): + root = zipfile.Path(alpharep) + assert not root.joinpath('missing.txt').is_file() + + @pass_alpharep + def test_iterdir_on_file(self, alpharep): + root = zipfile.Path(alpharep) + a, n, b, g, j = root.iterdir() + with self.assertRaises(ValueError): + a.iterdir() + + @pass_alpharep + def test_subdir_is_dir(self, alpharep): + root = zipfile.Path(alpharep) + assert (root / 'b').is_dir() + assert (root / 'b/').is_dir() + assert (root / 'g').is_dir() + assert (root / 'g/').is_dir() + + @pass_alpharep + def test_open(self, alpharep): + root = zipfile.Path(alpharep) + a, n, b, g, j = root.iterdir() + with a.open(encoding="utf-8") as strm: + data = strm.read() + self.assertEqual(data, "content of a") + with a.open('r', "utf-8") as strm: # not a kw, no gh-101144 TypeError + data = strm.read() + self.assertEqual(data, "content of a") + + def test_open_encoding_utf16(self): + in_memory_file = io.BytesIO() + zf = zipfile.ZipFile(in_memory_file, "w") + zf.writestr("path/16.txt", "This was utf-16".encode("utf-16")) + zf.filename = "test_open_utf16.zip" + root = zipfile.Path(zf) + (path,) = root.iterdir() + u16 = path.joinpath("16.txt") + with u16.open('r', "utf-16") as strm: + data = strm.read() + assert data == "This was utf-16" + with u16.open(encoding="utf-16") as strm: + data = strm.read() + assert data == "This was utf-16" + + def test_open_encoding_errors(self): + in_memory_file = io.BytesIO() + zf = zipfile.ZipFile(in_memory_file, "w") + zf.writestr("path/bad-utf8.bin", b"invalid utf-8: \xff\xff.") + zf.filename = "test_read_text_encoding_errors.zip" + root = zipfile.Path(zf) + (path,) = root.iterdir() + u16 = path.joinpath("bad-utf8.bin") + + # encoding= as a positional argument for gh-101144. + data = u16.read_text("utf-8", errors="ignore") + assert data == "invalid utf-8: ." + with u16.open("r", "utf-8", errors="surrogateescape") as f: + assert f.read() == "invalid utf-8: \udcff\udcff." + + # encoding= both positional and keyword is an error; gh-101144. + with self.assertRaisesRegex(TypeError, "encoding"): + data = u16.read_text("utf-8", encoding="utf-8") + + # both keyword arguments work. + with u16.open("r", encoding="utf-8", errors="strict") as f: + # error during decoding with wrong codec. + with self.assertRaises(UnicodeDecodeError): + f.read() + + @unittest.skipIf( + not getattr(sys.flags, 'warn_default_encoding', 0), + "Requires warn_default_encoding", + ) + @pass_alpharep + def test_encoding_warnings(self, alpharep): + """EncodingWarning must blame the read_text and open calls.""" + assert sys.flags.warn_default_encoding + root = zipfile.Path(alpharep) + with self.assertWarns(EncodingWarning) as wc: # noqa: F821 (astral-sh/ruff#13296) + root.joinpath("a.txt").read_text() + assert __file__ == wc.filename + with self.assertWarns(EncodingWarning) as wc: # noqa: F821 (astral-sh/ruff#13296) + root.joinpath("a.txt").open("r").close() + assert __file__ == wc.filename + + def test_open_write(self): + """ + If the zipfile is open for write, it should be possible to + write bytes or text to it. + """ + zf = zipfile.Path(zipfile.ZipFile(io.BytesIO(), mode='w')) + with zf.joinpath('file.bin').open('wb') as strm: + strm.write(b'binary contents') + with zf.joinpath('file.txt').open('w', encoding="utf-8") as strm: + strm.write('text file') + + @pass_alpharep + def test_open_extant_directory(self, alpharep): + """ + Attempting to open a directory raises IsADirectoryError. + """ + zf = zipfile.Path(alpharep) + with self.assertRaises(IsADirectoryError): + zf.joinpath('b').open() + + @pass_alpharep + def test_open_binary_invalid_args(self, alpharep): + root = zipfile.Path(alpharep) + with self.assertRaises(ValueError): + root.joinpath('a.txt').open('rb', encoding='utf-8') + with self.assertRaises(ValueError): + root.joinpath('a.txt').open('rb', 'utf-8') + + @pass_alpharep + def test_open_missing_directory(self, alpharep): + """ + Attempting to open a missing directory raises FileNotFoundError. + """ + zf = zipfile.Path(alpharep) + with self.assertRaises(FileNotFoundError): + zf.joinpath('z').open() + + @pass_alpharep + def test_read(self, alpharep): + root = zipfile.Path(alpharep) + a, n, b, g, j = root.iterdir() + assert a.read_text(encoding="utf-8") == "content of a" + # Also check positional encoding arg (gh-101144). + assert a.read_text("utf-8") == "content of a" + assert a.read_bytes() == b"content of a" + + @pass_alpharep + def test_joinpath(self, alpharep): + root = zipfile.Path(alpharep) + a = root.joinpath("a.txt") + assert a.is_file() + e = root.joinpath("b").joinpath("d").joinpath("e.txt") + assert e.read_text(encoding="utf-8") == "content of e" + + @pass_alpharep + def test_joinpath_multiple(self, alpharep): + root = zipfile.Path(alpharep) + e = root.joinpath("b", "d", "e.txt") + assert e.read_text(encoding="utf-8") == "content of e" + + @pass_alpharep + def test_traverse_truediv(self, alpharep): + root = zipfile.Path(alpharep) + a = root / "a.txt" + assert a.is_file() + e = root / "b" / "d" / "e.txt" + assert e.read_text(encoding="utf-8") == "content of e" + + @pass_alpharep + def test_pathlike_construction(self, alpharep): + """ + zipfile.Path should be constructable from a path-like object + """ + zipfile_ondisk = self.zipfile_ondisk(alpharep) + pathlike = FakePath(str(zipfile_ondisk)) + zipfile.Path(pathlike) + + @pass_alpharep + def test_traverse_pathlike(self, alpharep): + root = zipfile.Path(alpharep) + root / FakePath("a") + + @pass_alpharep + def test_parent(self, alpharep): + root = zipfile.Path(alpharep) + assert (root / 'a').parent.at == '' + assert (root / 'a' / 'b').parent.at == 'a/' + + @pass_alpharep + def test_dir_parent(self, alpharep): + root = zipfile.Path(alpharep) + assert (root / 'b').parent.at == '' + assert (root / 'b/').parent.at == '' + + @pass_alpharep + def test_missing_dir_parent(self, alpharep): + root = zipfile.Path(alpharep) + assert (root / 'missing dir/').parent.at == '' + + @pass_alpharep + def test_mutability(self, alpharep): + """ + If the underlying zipfile is changed, the Path object should + reflect that change. + """ + root = zipfile.Path(alpharep) + a, n, b, g, j = root.iterdir() + alpharep.writestr('foo.txt', 'foo') + alpharep.writestr('bar/baz.txt', 'baz') + assert any(child.name == 'foo.txt' for child in root.iterdir()) + assert (root / 'foo.txt').read_text(encoding="utf-8") == 'foo' + (baz,) = (root / 'bar').iterdir() + assert baz.read_text(encoding="utf-8") == 'baz' + + HUGE_ZIPFILE_NUM_ENTRIES = 2**13 + + def huge_zipfile(self): + """Create a read-only zipfile with a huge number of entries entries.""" + strm = io.BytesIO() + zf = zipfile.ZipFile(strm, "w") + for entry in map(str, range(self.HUGE_ZIPFILE_NUM_ENTRIES)): + zf.writestr(entry, entry) + zf.mode = 'r' + return zf + + def test_joinpath_constant_time(self): + """ + Ensure joinpath on items in zipfile is linear time. + """ + root = zipfile.Path(self.huge_zipfile()) + entries = jaraco.itertools.Counter(root.iterdir()) + for entry in entries: + entry.joinpath('suffix') + # Check the file iterated all items + assert entries.count == self.HUGE_ZIPFILE_NUM_ENTRIES + + @pass_alpharep + def test_read_does_not_close(self, alpharep): + alpharep = self.zipfile_ondisk(alpharep) + with zipfile.ZipFile(alpharep) as file: + for rep in range(2): + zipfile.Path(file, 'a.txt').read_text(encoding="utf-8") + + @pass_alpharep + def test_subclass(self, alpharep): + class Subclass(zipfile.Path): + pass + + root = Subclass(alpharep) + assert isinstance(root / 'b', Subclass) + + @pass_alpharep + def test_filename(self, alpharep): + root = zipfile.Path(alpharep) + assert root.filename == pathlib.Path('alpharep.zip') + + @pass_alpharep + def test_root_name(self, alpharep): + """ + The name of the root should be the name of the zipfile + """ + root = zipfile.Path(alpharep) + assert root.name == 'alpharep.zip' == root.filename.name + + @pass_alpharep + def test_root_on_disk(self, alpharep): + """ + The name/stem of the root should match the zipfile on disk. + + This condition must hold across platforms. + """ + root = zipfile.Path(self.zipfile_ondisk(alpharep)) + assert root.name == 'alpharep.zip' == root.filename.name + assert root.stem == 'alpharep' == root.filename.stem + + @pass_alpharep + def test_suffix(self, alpharep): + """ + The suffix of the root should be the suffix of the zipfile. + The suffix of each nested file is the final component's last suffix, if any. + Includes the leading period, just like pathlib.Path. + """ + root = zipfile.Path(alpharep) + assert root.suffix == '.zip' == root.filename.suffix + + b = root / "b.txt" + assert b.suffix == ".txt" + + c = root / "c" / "filename.tar.gz" + assert c.suffix == ".gz" + + d = root / "d" + assert d.suffix == "" + + @pass_alpharep + def test_suffixes(self, alpharep): + """ + The suffix of the root should be the suffix of the zipfile. + The suffix of each nested file is the final component's last suffix, if any. + Includes the leading period, just like pathlib.Path. + """ + root = zipfile.Path(alpharep) + assert root.suffixes == ['.zip'] == root.filename.suffixes + + b = root / 'b.txt' + assert b.suffixes == ['.txt'] + + c = root / 'c' / 'filename.tar.gz' + assert c.suffixes == ['.tar', '.gz'] + + d = root / 'd' + assert d.suffixes == [] + + e = root / '.hgrc' + assert e.suffixes == [] + + @pass_alpharep + def test_suffix_no_filename(self, alpharep): + alpharep.filename = None + root = zipfile.Path(alpharep) + assert root.joinpath('example').suffix == "" + assert root.joinpath('example').suffixes == [] + + @pass_alpharep + def test_stem(self, alpharep): + """ + The final path component, without its suffix + """ + root = zipfile.Path(alpharep) + assert root.stem == 'alpharep' == root.filename.stem + + b = root / "b.txt" + assert b.stem == "b" + + c = root / "c" / "filename.tar.gz" + assert c.stem == "filename.tar" + + d = root / "d" + assert d.stem == "d" + + assert (root / ".gitignore").stem == ".gitignore" + + @pass_alpharep + def test_root_parent(self, alpharep): + root = zipfile.Path(alpharep) + assert root.parent == pathlib.Path('.') + root.root.filename = 'foo/bar.zip' + assert root.parent == pathlib.Path('foo') + + @pass_alpharep + def test_root_unnamed(self, alpharep): + """ + It is an error to attempt to get the name + or parent of an unnamed zipfile. + """ + alpharep.filename = None + root = zipfile.Path(alpharep) + with self.assertRaises(TypeError): + root.name + with self.assertRaises(TypeError): + root.parent + + # .name and .parent should still work on subs + sub = root / "b" + assert sub.name == "b" + assert sub.parent + + @pass_alpharep + def test_match_and_glob(self, alpharep): + root = zipfile.Path(alpharep) + assert not root.match("*.txt") + + assert list(root.glob("b/c.*")) == [zipfile.Path(alpharep, "b/c.txt")] + assert list(root.glob("b/*.txt")) == [ + zipfile.Path(alpharep, "b/c.txt"), + zipfile.Path(alpharep, "b/f.txt"), + ] + + @pass_alpharep + def test_glob_recursive(self, alpharep): + root = zipfile.Path(alpharep) + files = root.glob("**/*.txt") + assert all(each.match("*.txt") for each in files) + + assert list(root.glob("**/*.txt")) == list(root.rglob("*.txt")) + + @pass_alpharep + def test_glob_dirs(self, alpharep): + root = zipfile.Path(alpharep) + assert list(root.glob('b')) == [zipfile.Path(alpharep, "b/")] + assert list(root.glob('b*')) == [zipfile.Path(alpharep, "b/")] + + @pass_alpharep + def test_glob_subdir(self, alpharep): + root = zipfile.Path(alpharep) + assert list(root.glob('g/h')) == [zipfile.Path(alpharep, "g/h/")] + assert list(root.glob('g*/h*')) == [zipfile.Path(alpharep, "g/h/")] + + @pass_alpharep + def test_glob_subdirs(self, alpharep): + root = zipfile.Path(alpharep) + + assert list(root.glob("*/i.txt")) == [] + assert list(root.rglob("*/i.txt")) == [zipfile.Path(alpharep, "g/h/i.txt")] + + @pass_alpharep + def test_glob_does_not_overmatch_dot(self, alpharep): + root = zipfile.Path(alpharep) + + assert list(root.glob("*.xt")) == [] + + @pass_alpharep + def test_glob_single_char(self, alpharep): + root = zipfile.Path(alpharep) + + assert list(root.glob("a?txt")) == [zipfile.Path(alpharep, "a.txt")] + assert list(root.glob("a[.]txt")) == [zipfile.Path(alpharep, "a.txt")] + assert list(root.glob("a[?]txt")) == [] + + @pass_alpharep + def test_glob_chars(self, alpharep): + root = zipfile.Path(alpharep) + + assert list(root.glob("j/?.b[ai][nz]")) == [ + zipfile.Path(alpharep, "j/k.bin"), + zipfile.Path(alpharep, "j/l.baz"), + ] + + def test_glob_empty(self): + root = zipfile.Path(zipfile.ZipFile(io.BytesIO(), 'w')) + with self.assertRaises(ValueError): + root.glob('') + + @pass_alpharep + def test_eq_hash(self, alpharep): + root = zipfile.Path(alpharep) + assert root == zipfile.Path(alpharep) + + assert root != (root / "a.txt") + assert (root / "a.txt") == (root / "a.txt") + + root = zipfile.Path(alpharep) + assert root in {root} + + @pass_alpharep + def test_is_symlink(self, alpharep): + root = zipfile.Path(alpharep) + assert not root.joinpath('a.txt').is_symlink() + assert root.joinpath('n.txt').is_symlink() + + @pass_alpharep + def test_relative_to(self, alpharep): + root = zipfile.Path(alpharep) + relative = root.joinpath("b", "c.txt").relative_to(root / "b") + assert str(relative) == "c.txt" + + relative = root.joinpath("b", "d", "e.txt").relative_to(root / "b") + assert str(relative) == "d/e.txt" + + @pass_alpharep + def test_inheritance(self, alpharep): + cls = type('PathChild', (zipfile.Path,), {}) + file = cls(alpharep).joinpath('some dir').parent + assert isinstance(file, cls) + + @unittest.skipIf(sys.platform == 'win32', "TODO: RUSTPYTHON, fails on Windows") + @parameterize( + ['alpharep', 'path_type', 'subpath'], + itertools.product( + alpharep_generators, + [str, FakePath], + ['', 'b/'], + ), + ) + def test_pickle(self, alpharep, path_type, subpath): + zipfile_ondisk = path_type(str(self.zipfile_ondisk(alpharep))) + + saved_1 = pickle.dumps(zipfile.Path(zipfile_ondisk, at=subpath)) + restored_1 = pickle.loads(saved_1) + first, *rest = restored_1.iterdir() + assert first.read_text(encoding='utf-8').startswith('content of ') + + @pass_alpharep + def test_extract_orig_with_implied_dirs(self, alpharep): + """ + A zip file wrapped in a Path should extract even with implied dirs. + """ + source_path = self.zipfile_ondisk(alpharep) + zf = zipfile.ZipFile(source_path) + # wrap the zipfile for its side effect + zipfile.Path(zf) + zf.extractall(source_path.parent) + + @pass_alpharep + def test_getinfo_missing(self, alpharep): + """ + Validate behavior of getinfo on original zipfile after wrapping. + """ + zipfile.Path(alpharep) + with self.assertRaises(KeyError): + alpharep.getinfo('does-not-exist') + + def test_malformed_paths(self): + """ + Path should handle malformed paths gracefully. + + Paths with leading slashes are not visible. + + Paths with dots are treated like regular files. + """ + data = io.BytesIO() + zf = zipfile.ZipFile(data, "w") + zf.writestr("/one-slash.txt", b"content") + zf.writestr("//two-slash.txt", b"content") + zf.writestr("../parent.txt", b"content") + zf.filename = '' + root = zipfile.Path(zf) + assert list(map(str, root.iterdir())) == ['../'] + assert root.joinpath('..').joinpath('parent.txt').read_bytes() == b'content' + + def test_unsupported_names(self): + """ + Path segments with special characters are readable. + + On some platforms or file systems, characters like + ``:`` and ``?`` are not allowed, but they are valid + in the zip file. + """ + data = io.BytesIO() + zf = zipfile.ZipFile(data, "w") + zf.writestr("path?", b"content") + zf.writestr("V: NMS.flac", b"fLaC...") + zf.filename = '' + root = zipfile.Path(zf) + contents = root.iterdir() + assert next(contents).name == 'path?' + assert next(contents).name == 'V: NMS.flac' + assert root.joinpath('V: NMS.flac').read_bytes() == b"fLaC..." + + def test_backslash_not_separator(self): + """ + In a zip file, backslashes are not separators. + """ + data = io.BytesIO() + zf = zipfile.ZipFile(data, "w") + zf.writestr(DirtyZipInfo.for_name("foo\\bar", zf), b"content") + zf.filename = '' + root = zipfile.Path(zf) + (first,) = root.iterdir() + assert not first.is_dir() + assert first.name == 'foo\\bar' + + @pass_alpharep + def test_interface(self, alpharep): + from importlib.resources.abc import Traversable + + zf = zipfile.Path(alpharep) + assert isinstance(zf, Traversable) + + +class DirtyZipInfo(zipfile.ZipInfo): + """ + Bypass name sanitization. + """ + + def __init__(self, filename, *args, **kwargs): + super().__init__(filename, *args, **kwargs) + self.filename = filename + + @classmethod + def for_name(cls, name, archive): + """ + Construct the same way that ZipFile.writestr does. + + TODO: extract this functionality and re-use + """ + self = cls(filename=name, date_time=time.localtime(time.time())[:6]) + self.compress_type = archive.compression + self.compress_level = archive.compresslevel + if self.filename.endswith('/'): # pragma: no cover + self.external_attr = 0o40775 << 16 # drwxrwxr-x + self.external_attr |= 0x10 # MS-DOS directory flag + else: + self.external_attr = 0o600 << 16 # ?rw------- + return self diff --git a/Lib/test/test_zipfile/_path/write-alpharep.py b/Lib/test/test_zipfile/_path/write-alpharep.py new file mode 100644 index 00000000000..7418391abad --- /dev/null +++ b/Lib/test/test_zipfile/_path/write-alpharep.py @@ -0,0 +1,3 @@ +from . import test_path + +__name__ == '__main__' and test_path.build_alpharep_fixture().extractall('alpharep') diff --git a/Lib/test/test_zipfile.py b/Lib/test/test_zipfile/test_core.py similarity index 72% rename from Lib/test/test_zipfile.py rename to Lib/test/test_zipfile/test_core.py index e7705590cd2..fa1feef00cd 100644 --- a/Lib/test/test_zipfile.py +++ b/Lib/test/test_zipfile/test_core.py @@ -1,12 +1,11 @@ +import _pyio import array import contextlib import importlib.util import io import itertools import os -import pathlib import posixpath -import string import struct import subprocess import sys @@ -14,16 +13,20 @@ import unittest import unittest.mock as mock import zipfile -import functools from tempfile import TemporaryFile from random import randint, random, randbytes +from test import archiver_tests from test.support import script_helper -from test.support import (findfile, requires_zlib, requires_bz2, - requires_lzma, captured_stdout) -from test.support.os_helper import TESTFN, unlink, rmtree, temp_dir, temp_cwd +from test.support import ( + findfile, requires_zlib, requires_bz2, requires_lzma, + captured_stdout, captured_stderr, requires_subprocess +) +from test.support.os_helper import ( + TESTFN, unlink, rmtree, temp_dir, temp_cwd, fd_count, FakePath +) TESTFN2 = TESTFN + "2" @@ -120,8 +123,9 @@ def zip_test(self, f, compression, compresslevel=None): self.assertEqual(info.filename, nm) self.assertEqual(info.file_size, len(self.data)) - # Check that testzip doesn't raise an exception - zipfp.testzip() + # Check that testzip thinks the archive is ok + # (it returns None if all contents could be read properly) + self.assertIsNone(zipfp.testzip()) def test_basic(self): for f in get_files(self): @@ -156,7 +160,7 @@ def test_open(self): self.zip_open_test(f, self.compression) def test_open_with_pathlike(self): - path = pathlib.Path(TESTFN2) + path = FakePath(TESTFN2) self.zip_open_test(path, self.compression) with zipfile.ZipFile(path, "r", self.compression) as zipfp: self.assertIsInstance(zipfp.filename, str) @@ -312,7 +316,7 @@ def test_writestr_compresslevel(self): # Compression level follows the constructor. a_info = zipfp.getinfo('a.txt') self.assertEqual(a_info.compress_type, self.compression) - self.assertEqual(a_info._compresslevel, 1) + self.assertEqual(a_info.compress_level, 1) # Compression level is overridden. b_info = zipfp.getinfo('b.txt') @@ -386,7 +390,6 @@ def test_repr(self): with zipfp.open(fname) as zipopen: r = repr(zipopen) self.assertIn('name=%r' % fname, r) - self.assertIn("mode='r'", r) if self.compression != zipfile.ZIP_STORED: self.assertIn('compress_type=', r) self.assertIn('[closed]', repr(zipopen)) @@ -405,7 +408,7 @@ def test_per_file_compresslevel(self): one_info = zipfp.getinfo('compress_1') nine_info = zipfp.getinfo('compress_9') self.assertEqual(one_info._compresslevel, 1) - self.assertEqual(nine_info._compresslevel, 9) + self.assertEqual(nine_info.compress_level, 9) def test_writing_errors(self): class BrokenFile(io.BytesIO): @@ -443,6 +446,27 @@ def write(self, data): self.assertEqual(zipfp.read('file1'), b'data1') self.assertEqual(zipfp.read('file2'), b'data2') + def test_zipextfile_attrs(self): + fname = "somefile.txt" + with zipfile.ZipFile(TESTFN2, mode="w") as zipfp: + zipfp.writestr(fname, "bogus") + + with zipfile.ZipFile(TESTFN2, mode="r") as zipfp: + with zipfp.open(fname) as fid: + self.assertEqual(fid.name, fname) + self.assertRaises(io.UnsupportedOperation, fid.fileno) + self.assertEqual(fid.mode, 'rb') + self.assertIs(fid.readable(), True) + self.assertIs(fid.writable(), False) + self.assertIs(fid.seekable(), True) + self.assertIs(fid.closed, False) + self.assertIs(fid.closed, True) + self.assertEqual(fid.name, fname) + self.assertEqual(fid.mode, 'rb') + self.assertRaises(io.UnsupportedOperation, fid.fileno) + self.assertRaises(ValueError, fid.readable) + self.assertIs(fid.writable(), False) + self.assertRaises(ValueError, fid.seekable) def tearDown(self): unlink(TESTFN) @@ -574,17 +598,16 @@ def test_write_default_name(self): def test_io_on_closed_zipextfile(self): fname = "somefile.txt" - with zipfile.ZipFile(TESTFN2, mode="w") as zipfp: + with zipfile.ZipFile(TESTFN2, mode="w", compression=self.compression) as zipfp: zipfp.writestr(fname, "bogus") with zipfile.ZipFile(TESTFN2, mode="r") as zipfp: with zipfp.open(fname) as fid: fid.close() + self.assertIs(fid.closed, True) self.assertRaises(ValueError, fid.read) self.assertRaises(ValueError, fid.seek, 0) self.assertRaises(ValueError, fid.tell) - self.assertRaises(ValueError, fid.readable) - self.assertRaises(ValueError, fid.seekable) def test_write_to_readonly(self): """Check that trying to call write() on a readonly ZipFile object @@ -744,8 +767,8 @@ def zip_test(self, f, compression): self.assertEqual(info.filename, nm) self.assertEqual(info.file_size, len(self.data)) - # Check that testzip doesn't raise an exception - zipfp.testzip() + # Check that testzip thinks the archive is valid + self.assertIsNone(zipfp.testzip()) def test_basic(self): for f in get_files(self): @@ -1077,6 +1100,159 @@ def test_generated_valid_zip64_extra(self): self.assertEqual(zinfo.header_offset, expected_header_offset) self.assertEqual(zf.read(zinfo), expected_content) + def test_force_zip64(self): + """Test that forcing zip64 extensions correctly notes this in the zip file""" + + # GH-103861 describes an issue where forcing a small file to use zip64 + # extensions would add a zip64 extra record, but not change the data + # sizes to 0xFFFFFFFF to indicate to the extractor that the zip64 + # record should be read. Additionally, it would not set the required + # version to indicate that zip64 extensions are required to extract it. + # This test replicates the situation and reads the raw data to specifically ensure: + # - The required extract version is always >= ZIP64_VERSION + # - The compressed and uncompressed size in the file headers are both + # 0xFFFFFFFF (ie. point to zip64 record) + # - The zip64 record is provided and has the correct sizes in it + # Other aspects of the zip are checked as well, but verifying the above is the main goal. + # Because this is hard to verify by parsing the data as a zip, the raw + # bytes are checked to ensure that they line up with the zip spec. + # The spec for this can be found at: https://pkware.cachefly.net/webdocs/casestudies/APPNOTE.TXT + # The relevant sections for this test are: + # - 4.3.7 for local file header + # - 4.5.3 for zip64 extra field + + data = io.BytesIO() + with zipfile.ZipFile(data, mode="w", allowZip64=True) as zf: + with zf.open("text.txt", mode="w", force_zip64=True) as zi: + zi.write(b"_") + + zipdata = data.getvalue() + + # pull out and check zip information + ( + header, vers, os, flags, comp, csize, usize, fn_len, + ex_total_len, filename, ex_id, ex_len, ex_usize, ex_csize, cd_sig + ) = struct.unpack("<4sBBHH8xIIHH8shhQQx4s", zipdata[:63]) + + self.assertEqual(header, b"PK\x03\x04") # local file header + self.assertGreaterEqual(vers, zipfile.ZIP64_VERSION) # requires zip64 to extract + self.assertEqual(os, 0) # compatible with MS-DOS + self.assertEqual(flags, 0) # no flags + self.assertEqual(comp, 0) # compression method = stored + self.assertEqual(csize, 0xFFFFFFFF) # sizes are in zip64 extra + self.assertEqual(usize, 0xFFFFFFFF) + self.assertEqual(fn_len, 8) # filename len + self.assertEqual(ex_total_len, 20) # size of extra records + self.assertEqual(ex_id, 1) # Zip64 extra record + self.assertEqual(ex_len, 16) # 16 bytes of data + self.assertEqual(ex_usize, 1) # uncompressed size + self.assertEqual(ex_csize, 1) # compressed size + self.assertEqual(cd_sig, b"PK\x01\x02") # ensure the central directory header is next + + z = zipfile.ZipFile(io.BytesIO(zipdata)) + zinfos = z.infolist() + self.assertEqual(len(zinfos), 1) + self.assertGreaterEqual(zinfos[0].extract_version, zipfile.ZIP64_VERSION) # requires zip64 to extract + + def test_unseekable_zip_unknown_filesize(self): + """Test that creating a zip with/without seeking will raise a RuntimeError if zip64 was required but not used""" + + def make_zip(fp): + with zipfile.ZipFile(fp, mode="w", allowZip64=True) as zf: + with zf.open("text.txt", mode="w", force_zip64=False) as zi: + zi.write(b"_" * (zipfile.ZIP64_LIMIT + 1)) + + self.assertRaises(RuntimeError, make_zip, io.BytesIO()) + self.assertRaises(RuntimeError, make_zip, Unseekable(io.BytesIO())) + + def test_zip64_required_not_allowed_fail(self): + """Test that trying to add a large file to a zip that doesn't allow zip64 extensions fails on add""" + def make_zip(fp): + with zipfile.ZipFile(fp, mode="w", allowZip64=False) as zf: + # pretend zipfile.ZipInfo.from_file was used to get the name and filesize + info = zipfile.ZipInfo("text.txt") + info.file_size = zipfile.ZIP64_LIMIT + 1 + zf.open(info, mode="w") + + self.assertRaises(zipfile.LargeZipFile, make_zip, io.BytesIO()) + self.assertRaises(zipfile.LargeZipFile, make_zip, Unseekable(io.BytesIO())) + + def test_unseekable_zip_known_filesize(self): + """Test that creating a zip without seeking will use zip64 extensions if the file size is provided up-front""" + + # This test ensures that the zip will use a zip64 data descriptor (same + # as a regular data descriptor except the sizes are 8 bytes instead of + # 4) record to communicate the size of a file if the zip is being + # written to an unseekable stream. + # Because this sort of thing is hard to verify by parsing the data back + # in as a zip, this test looks at the raw bytes created to ensure that + # the correct data has been generated. + # The spec for this can be found at: https://pkware.cachefly.net/webdocs/casestudies/APPNOTE.TXT + # The relevant sections for this test are: + # - 4.3.7 for local file header + # - 4.3.9 for the data descriptor + # - 4.5.3 for zip64 extra field + + file_size = zipfile.ZIP64_LIMIT + 1 + + def make_zip(fp): + with zipfile.ZipFile(fp, mode="w", allowZip64=True) as zf: + # pretend zipfile.ZipInfo.from_file was used to get the name and filesize + info = zipfile.ZipInfo("text.txt") + info.file_size = file_size + with zf.open(info, mode="w", force_zip64=False) as zi: + zi.write(b"_" * file_size) + return fp + + # check seekable file information + seekable_data = make_zip(io.BytesIO()).getvalue() + ( + header, vers, os, flags, comp, csize, usize, fn_len, + ex_total_len, filename, ex_id, ex_len, ex_usize, ex_csize, + cd_sig + ) = struct.unpack("<4sBBHH8xIIHH8shhQQ{}x4s".format(file_size), seekable_data[:62 + file_size]) + + self.assertEqual(header, b"PK\x03\x04") # local file header + self.assertGreaterEqual(vers, zipfile.ZIP64_VERSION) # requires zip64 to extract + self.assertEqual(os, 0) # compatible with MS-DOS + self.assertEqual(flags, 0) # no flags set + self.assertEqual(comp, 0) # compression method = stored + self.assertEqual(csize, 0xFFFFFFFF) # sizes are in zip64 extra + self.assertEqual(usize, 0xFFFFFFFF) + self.assertEqual(fn_len, 8) # filename len + self.assertEqual(ex_total_len, 20) # size of extra records + self.assertEqual(ex_id, 1) # Zip64 extra record + self.assertEqual(ex_len, 16) # 16 bytes of data + self.assertEqual(ex_usize, file_size) # uncompressed size + self.assertEqual(ex_csize, file_size) # compressed size + self.assertEqual(cd_sig, b"PK\x01\x02") # ensure the central directory header is next + + # check unseekable file information + unseekable_data = make_zip(Unseekable(io.BytesIO())).fp.getvalue() + ( + header, vers, os, flags, comp, csize, usize, fn_len, + ex_total_len, filename, ex_id, ex_len, ex_usize, ex_csize, + dd_header, dd_usize, dd_csize, cd_sig + ) = struct.unpack("<4sBBHH8xIIHH8shhQQ{}x4s4xQQ4s".format(file_size), unseekable_data[:86 + file_size]) + + self.assertEqual(header, b"PK\x03\x04") # local file header + self.assertGreaterEqual(vers, zipfile.ZIP64_VERSION) # requires zip64 to extract + self.assertEqual(os, 0) # compatible with MS-DOS + self.assertEqual("{:b}".format(flags), "1000") # streaming flag set + self.assertEqual(comp, 0) # compression method = stored + self.assertEqual(csize, 0xFFFFFFFF) # sizes are in zip64 extra + self.assertEqual(usize, 0xFFFFFFFF) + self.assertEqual(fn_len, 8) # filename len + self.assertEqual(ex_total_len, 20) # size of extra records + self.assertEqual(ex_id, 1) # Zip64 extra record + self.assertEqual(ex_len, 16) # 16 bytes of data + self.assertEqual(ex_usize, 0) # uncompressed size - 0 to defer to data descriptor + self.assertEqual(ex_csize, 0) # compressed size - 0 to defer to data descriptor + self.assertEqual(dd_header, b"PK\07\x08") # data descriptor + self.assertEqual(dd_usize, file_size) # file size (8 bytes because zip64) + self.assertEqual(dd_csize, file_size) # compressed size (8 bytes because zip64) + self.assertEqual(cd_sig, b"PK\x01\x02") # ensure the central directory header is next + @requires_zlib() class DeflateTestZip64InSmallFiles(AbstractTestZip64InSmallFiles, @@ -1128,6 +1304,25 @@ def test_issue44439(self): self.assertEqual(data.write(q), LENGTH) self.assertEqual(zip.getinfo('data').file_size, LENGTH) + def test_zipwritefile_attrs(self): + fname = "somefile.txt" + with zipfile.ZipFile(TESTFN2, mode="w", compression=self.compression) as zipfp: + with zipfp.open(fname, 'w') as fid: + self.assertEqual(fid.name, fname) + self.assertRaises(io.UnsupportedOperation, fid.fileno) + self.assertEqual(fid.mode, 'wb') + self.assertIs(fid.readable(), False) + self.assertIs(fid.writable(), True) + self.assertIs(fid.seekable(), False) + self.assertIs(fid.closed, False) + self.assertIs(fid.closed, True) + self.assertEqual(fid.name, fname) + self.assertEqual(fid.mode, 'wb') + self.assertRaises(io.UnsupportedOperation, fid.fileno) + self.assertIs(fid.readable(), False) + self.assertIs(fid.writable(), True) + self.assertIs(fid.seekable(), False) + class StoredWriterTests(AbstractWriterTests, unittest.TestCase): compression = zipfile.ZIP_STORED @@ -1210,7 +1405,7 @@ def test_write_python_package(self): self.assertCompiledIn('email/__init__.py', names) self.assertCompiledIn('email/mime/text.py', names) - # TODO: RUSTPYTHON + # TODO: RUSTPYTHON - AttributeError: module 'os' has no attribute 'supports_effective_ids' @unittest.expectedFailure def test_write_filtered_python_package(self): import test @@ -1338,7 +1533,7 @@ def test_write_pathlike(self): fp.write("print(42)\n") with TemporaryFile() as t, zipfile.PyZipFile(t, "w") as zipfp: - zipfp.writepy(pathlib.Path(TESTFN2) / "mod1.py") + zipfp.writepy(FakePath(os.path.join(TESTFN2, "mod1.py"))) names = zipfp.namelist() self.assertCompiledIn('mod1.py', names) finally: @@ -1396,7 +1591,7 @@ def test_extract_with_target(self): def test_extract_with_target_pathlike(self): with temp_dir() as extdir: - self._test_extract_with_target(pathlib.Path(extdir)) + self._test_extract_with_target(FakePath(extdir)) def test_extract_all(self): with temp_cwd(): @@ -1431,7 +1626,7 @@ def test_extract_all_with_target(self): def test_extract_all_with_target_pathlike(self): with temp_dir() as extdir: - self._test_extract_all_with_target(pathlib.Path(extdir)) + self._test_extract_all_with_target(FakePath(extdir)) def check_file(self, filename, content): self.assertTrue(os.path.isfile(filename)) @@ -1444,6 +1639,8 @@ def test_sanitize_windows_name(self): self.assertEqual(san(r',,?,C:,foo,bar/z', ','), r'_,C_,foo,bar/z') self.assertEqual(san(r'a\b,ce|f"g?h*i', ','), r'a\b,c_d_e_f_g_h_i') self.assertEqual(san('../../foo../../ba..r', '/'), r'foo/ba..r') + self.assertEqual(san(' / /foo / /ba r', '/'), r'foo/ba r') + self.assertEqual(san(' . /. /foo ./ . /. ./ba .r', '/'), r'foo/ba .r') def test_extract_hackers_arcnames_common_cases(self): common_hacknames = [ @@ -1458,8 +1655,6 @@ def test_extract_hackers_arcnames_common_cases(self): ] self._test_extract_hackers_arcnames(common_hacknames) - # TODO: RUSTPYTHON - @unittest.expectedFailure @unittest.skipIf(os.path.sep != '\\', 'Requires \\ as path separator.') def test_extract_hackers_arcnames_windows_only(self): """Test combination of path fixing and windows name sanitization.""" @@ -1474,10 +1669,10 @@ def test_extract_hackers_arcnames_windows_only(self): (r'C:\foo\bar', 'foo/bar'), (r'//conky/mountpoint/foo/bar', 'foo/bar'), (r'\\conky\mountpoint\foo\bar', 'foo/bar'), - (r'///conky/mountpoint/foo/bar', 'conky/mountpoint/foo/bar'), - (r'\\\conky\mountpoint\foo\bar', 'conky/mountpoint/foo/bar'), - (r'//conky//mountpoint/foo/bar', 'conky/mountpoint/foo/bar'), - (r'\\conky\\mountpoint\foo\bar', 'conky/mountpoint/foo/bar'), + (r'///conky/mountpoint/foo/bar', 'mountpoint/foo/bar'), + (r'\\\conky\mountpoint\foo\bar', 'mountpoint/foo/bar'), + (r'//conky//mountpoint/foo/bar', 'mountpoint/foo/bar'), + (r'\\conky\\mountpoint\foo\bar', 'mountpoint/foo/bar'), (r'//?/C:/foo/bar', 'foo/bar'), (r'\\?\C:\foo\bar', 'foo/bar'), (r'C:/../C:/foo/bar', 'C_/foo/bar'), @@ -1539,6 +1734,33 @@ def _test_extract_hackers_arcnames(self, hacknames): unlink(TESTFN2) +class OverwriteTests(archiver_tests.OverwriteTests, unittest.TestCase): + testdir = TESTFN + + @classmethod + def setUpClass(cls): + p = cls.ar_with_file = TESTFN + '-with-file.zip' + cls.addClassCleanup(unlink, p) + with zipfile.ZipFile(p, 'w') as zipfp: + zipfp.writestr('test', b'newcontent') + + p = cls.ar_with_dir = TESTFN + '-with-dir.zip' + cls.addClassCleanup(unlink, p) + with zipfile.ZipFile(p, 'w') as zipfp: + zipfp.mkdir('test') + + p = cls.ar_with_implicit_dir = TESTFN + '-with-implicit-dir.zip' + cls.addClassCleanup(unlink, p) + with zipfile.ZipFile(p, 'w') as zipfp: + zipfp.writestr('test/file', b'newcontent') + + def open(self, path): + return zipfile.ZipFile(path, 'r') + + def extractall(self, ar): + ar.extractall(self.testdir) + + class OtherTests(unittest.TestCase): def test_open_via_zip_info(self): # Create the ZIP archive @@ -1564,7 +1786,7 @@ def test_writestr_extended_local_header_issue1202(self): with zipfile.ZipFile(TESTFN2, 'w') as orig_zip: for data in 'abcdefghijklmnop': zinfo = zipfile.ZipInfo(data) - zinfo.flag_bits |= 0x08 # Include an extended local header. + zinfo.flag_bits |= zipfile._MASK_USE_DATA_DESCRIPTOR # Include an extended local header. orig_zip.writestr(zinfo, data) def test_close(self): @@ -1606,7 +1828,7 @@ def test_unsupported_version(self): @requires_zlib() def test_read_unicode_filenames(self): # bug #10801 - fname = findfile('zip_cp437_header.zip') + fname = findfile('zip_cp437_header.zip', subdir='archivetestdata') with zipfile.ZipFile(fname) as zipfp: for name in zipfp.namelist(): zipfp.open(name).close() @@ -1621,6 +1843,44 @@ def test_write_unicode_filenames(self): self.assertEqual(zf.filelist[0].filename, "foo.txt") self.assertEqual(zf.filelist[1].filename, "\xf6.txt") + def create_zipfile_with_extra_data(self, filename, extra_data_name): + with zipfile.ZipFile(TESTFN, mode='w') as zf: + filename_encoded = filename.encode("utf-8") + # create a ZipInfo object with Unicode path extra field + zip_info = zipfile.ZipInfo(filename) + + tag_for_unicode_path = b'\x75\x70' + version_of_unicode_path = b'\x01' + + import zlib + filename_crc = struct.pack('= 0: # found the magic number; attempt to unpack and interpret @@ -323,8 +352,26 @@ def _EndRecData(fpin): # Unable to find a valid end of central directory structure return None - -class ZipInfo (object): +def _sanitize_filename(filename): + """Terminate the file name at the first null byte and + ensure paths always use forward slashes as the directory separator.""" + + # Terminate the file name at the first null byte. Null bytes in file + # names are used as tricks by viruses in archives. + null_byte = filename.find(chr(0)) + if null_byte >= 0: + filename = filename[0:null_byte] + # This is used to ensure paths in generated ZIP files always use + # forward slashes as the directory separator, as required by the + # ZIP format specification. + if os.sep != "/" and os.sep in filename: + filename = filename.replace(os.sep, "/") + if os.altsep and os.altsep != "/" and os.altsep in filename: + filename = filename.replace(os.altsep, "/") + return filename + + +class ZipInfo: """Class with attributes describing each file in the ZIP archive.""" __slots__ = ( @@ -332,7 +379,7 @@ class ZipInfo (object): 'filename', 'date_time', 'compress_type', - '_compresslevel', + 'compress_level', 'comment', 'extra', 'create_system', @@ -348,21 +395,15 @@ class ZipInfo (object): 'compress_size', 'file_size', '_raw_time', + '_end_offset', ) def __init__(self, filename="NoName", date_time=(1980,1,1,0,0,0)): self.orig_filename = filename # Original file name in archive - # Terminate the file name at the first null byte. Null bytes in file - # names are used as tricks by viruses in archives. - null_byte = filename.find(chr(0)) - if null_byte >= 0: - filename = filename[0:null_byte] - # This is used to ensure paths in generated ZIP files always use - # forward slashes as the directory separator, as required by the - # ZIP format specification. - if os.sep != "/" and os.sep in filename: - filename = filename.replace(os.sep, "/") + # Terminate the file name at the first null byte and + # ensure paths always use forward slashes as the directory separator. + filename = _sanitize_filename(filename) self.filename = filename # Normalized file name self.date_time = date_time # year, month, day, hour, min, sec @@ -372,7 +413,7 @@ def __init__(self, filename="NoName", date_time=(1980,1,1,0,0,0)): # Standard values: self.compress_type = ZIP_STORED # Type of compression for the file - self._compresslevel = None # Level for the compressor + self.compress_level = None # Level for the compressor self.comment = b"" # Comment for each file self.extra = b"" # ZIP extra data if sys.platform == 'win32': @@ -389,10 +430,20 @@ def __init__(self, filename="NoName", date_time=(1980,1,1,0,0,0)): self.external_attr = 0 # External file attributes self.compress_size = 0 # Size of the compressed file self.file_size = 0 # Size of the uncompressed file + self._end_offset = None # Start of the next local header or central directory # Other attributes are set by class ZipFile: # header_offset Byte offset to the file header # CRC CRC-32 of the uncompressed file + # Maintain backward compatibility with the old protected attribute name. + @property + def _compresslevel(self): + return self.compress_level + + @_compresslevel.setter + def _compresslevel(self, value): + self.compress_level = value + def __repr__(self): result = ['<%s filename=%r' % (self.__class__.__name__, self.filename)] if self.compress_type != ZIP_STORED: @@ -416,11 +467,16 @@ def __repr__(self): return ''.join(result) def FileHeader(self, zip64=None): - """Return the per-file header as a bytes object.""" + """Return the per-file header as a bytes object. + + When the optional zip64 arg is None rather than a bool, we will + decide based upon the file_size and compress_size, if known, + False otherwise. + """ dt = self.date_time dosdate = (dt[0] - 1980) << 9 | dt[1] << 5 | dt[2] dostime = dt[3] << 11 | dt[4] << 5 | (dt[5] // 2) - if self.flag_bits & 0x08: + if self.flag_bits & _MASK_USE_DATA_DESCRIPTOR: # Set these to zero because we write them after the file data CRC = compress_size = file_size = 0 else: @@ -432,16 +488,13 @@ def FileHeader(self, zip64=None): min_version = 0 if zip64 is None: + # We always explicitly pass zip64 within this module.... This + # remains for anyone using ZipInfo.FileHeader as a public API. zip64 = file_size > ZIP64_LIMIT or compress_size > ZIP64_LIMIT if zip64: fmt = ' ZIP64_LIMIT or compress_size > ZIP64_LIMIT: - if not zip64: - raise LargeZipFile("Filesize would require ZIP64 extensions") - # File is larger than what fits into a 4 byte integer, - # fall back to the ZIP64 extension file_size = 0xffffffff compress_size = 0xffffffff min_version = ZIP64_VERSION @@ -465,9 +518,9 @@ def _encodeFilenameFlags(self): try: return self.filename.encode('ascii'), self.flag_bits except UnicodeEncodeError: - return self.filename.encode('utf-8'), self.flag_bits | 0x800 + return self.filename.encode('utf-8'), self.flag_bits | _MASK_UTF_FILENAME - def _decodeExtra(self): + def _decodeExtra(self, filename_crc): # Try to decode the extra field. extra = self.extra unpack = struct.unpack @@ -493,6 +546,22 @@ def _decodeExtra(self): except struct.error: raise BadZipFile(f"Corrupt zip64 extra field. " f"{field} not found.") from None + elif tp == 0x7075: + data = extra[4:ln+4] + # Unicode Path Extra Field + try: + up_version, up_name_crc = unpack('> 8) & 0xff else: @@ -862,7 +943,7 @@ def __repr__(self): result = ['<%s.%s' % (self.__class__.__module__, self.__class__.__qualname__)] if not self.closed: - result.append(' name=%r mode=%r' % (self.name, self.mode)) + result.append(' name=%r' % (self.name,)) if self._compress_type != ZIP_STORED: result.append(' compress_type=%s' % compressor_names.get(self._compress_type, @@ -1052,17 +1133,17 @@ def seekable(self): raise ValueError("I/O operation on closed file.") return self._seekable - def seek(self, offset, whence=0): + def seek(self, offset, whence=os.SEEK_SET): if self.closed: raise ValueError("seek on closed file.") if not self._seekable: raise io.UnsupportedOperation("underlying stream is not seekable") curr_pos = self.tell() - if whence == 0: # Seek from start of file + if whence == os.SEEK_SET: new_pos = offset - elif whence == 1: # Seek from current position + elif whence == os.SEEK_CUR: new_pos = curr_pos + offset - elif whence == 2: # Seek from EOF + elif whence == os.SEEK_END: new_pos = self._orig_file_size + offset else: raise ValueError("whence must be os.SEEK_SET (0), " @@ -1081,10 +1162,25 @@ def seek(self, offset, whence=0): # Just move the _offset index if the new position is in the _readbuffer self._offset = buff_offset read_offset = 0 + # Fast seek uncompressed unencrypted file + elif self._compress_type == ZIP_STORED and self._decrypter is None and read_offset != 0: + # disable CRC checking after first seeking - it would be invalid + self._expected_crc = None + # seek actual file taking already buffered data into account + read_offset -= len(self._readbuffer) - self._offset + self._fileobj.seek(read_offset, os.SEEK_CUR) + self._left -= read_offset + self._compress_left -= read_offset + self._eof = self._left <= 0 + read_offset = 0 + # flush read buffer + self._readbuffer = b'' + self._offset = 0 elif read_offset < 0: # Position is before the current position. Reset the ZipExtFile self._fileobj.seek(self._orig_compress_start) self._running_crc = self._orig_start_crc + self._expected_crc = self._orig_crc self._compress_left = self._orig_compress_size self._left = self._orig_file_size self._readbuffer = b'' @@ -1117,7 +1213,7 @@ def __init__(self, zf, zinfo, zip64): self._zip64 = zip64 self._zipfile = zf self._compressor = _get_compressor(zinfo.compress_type, - zinfo._compresslevel) + zinfo.compress_level) self._file_size = 0 self._compress_size = 0 self._crc = 0 @@ -1126,6 +1222,14 @@ def __init__(self, zf, zinfo, zip64): def _fileobj(self): return self._zipfile.fp + @property + def name(self): + return self._zinfo.filename + + @property + def mode(self): + return 'wb' + def writable(self): return True @@ -1164,21 +1268,20 @@ def close(self): self._zinfo.CRC = self._crc self._zinfo.file_size = self._file_size + if not self._zip64: + if self._file_size > ZIP64_LIMIT: + raise RuntimeError("File size too large, try using force_zip64") + if self._compress_size > ZIP64_LIMIT: + raise RuntimeError("Compressed size too large, try using force_zip64") + # Write updated header info - if self._zinfo.flag_bits & 0x08: + if self._zinfo.flag_bits & _MASK_USE_DATA_DESCRIPTOR: # Write CRC and file sizes after the file data fmt = ' ZIP64_LIMIT: - raise RuntimeError( - 'File size unexpectedly exceeded ZIP64 limit') - if self._compress_size > ZIP64_LIMIT: - raise RuntimeError( - 'Compressed size unexpectedly exceeded ZIP64 limit') # Seek backwards and write file header (which will now include # correct CRC and file sizes) @@ -1223,7 +1326,7 @@ class ZipFile: _windows_illegal_name_trans_table = None def __init__(self, file, mode="r", compression=ZIP_STORED, allowZip64=True, - compresslevel=None, *, strict_timestamps=True): + compresslevel=None, *, strict_timestamps=True, metadata_encoding=None): """Open the ZIP file with mode read 'r', write 'w', exclusive create 'x', or append 'a'.""" if mode not in ('r', 'w', 'x', 'a'): @@ -1242,6 +1345,12 @@ def __init__(self, file, mode="r", compression=ZIP_STORED, allowZip64=True, self.pwd = None self._comment = b'' self._strict_timestamps = strict_timestamps + self.metadata_encoding = metadata_encoding + + # Check that we don't try to write with nonconforming codecs + if self.metadata_encoding and mode != 'r': + raise ValueError( + "metadata_encoding is only supported for reading files") # Check if we were passed a file-like object if isinstance(file, os.PathLike): @@ -1374,13 +1483,14 @@ def _RealGetContents(self): if self.debug > 2: print(centdir) filename = fp.read(centdir[_CD_FILENAME_LENGTH]) - flags = centdir[5] - if flags & 0x800: + orig_filename_crc = crc32(filename) + flags = centdir[_CD_FLAG_BITS] + if flags & _MASK_UTF_FILENAME: # UTF-8 file names extension filename = filename.decode('utf-8') else: # Historical ZIP filename encoding - filename = filename.decode('cp437') + filename = filename.decode(self.metadata_encoding or 'cp437') # Create ZipInfo instance to store file information x = ZipInfo(filename) x.extra = fp.read(centdir[_CD_EXTRA_FIELD_LENGTH]) @@ -1397,8 +1507,7 @@ def _RealGetContents(self): x._raw_time = t x.date_time = ( (d>>9)+1980, (d>>5)&0xF, d&0x1F, t>>11, (t>>5)&0x3F, (t&0x1F) * 2 ) - - x._decodeExtra() + x._decodeExtra(orig_filename_crc) x.header_offset = x.header_offset + concat self.filelist.append(x) self.NameToInfo[x.filename] = x @@ -1411,6 +1520,11 @@ def _RealGetContents(self): if self.debug > 2: print("total", total) + end_offset = self.start_dir + for zinfo in reversed(sorted(self.filelist, + key=lambda zinfo: zinfo.header_offset)): + zinfo._end_offset = end_offset + end_offset = zinfo.header_offset def namelist(self): """Return a list of file names in the archive.""" @@ -1431,7 +1545,10 @@ def printdir(self, file=None): file=file) def testzip(self): - """Read all the files and check the CRC.""" + """Read all the files and check the CRC. + + Return None if all files could be read successfully, or the name + of the offending file otherwise.""" chunk_size = 2 ** 20 for zinfo in self.filelist: try: @@ -1480,7 +1597,8 @@ def comment(self, comment): self._didModify = True def read(self, name, pwd=None): - """Return file bytes for name.""" + """Return file bytes for name. 'pwd' is the password to decrypt + encrypted files.""" with self.open(name, "r", pwd) as fp: return fp.read() @@ -1502,8 +1620,6 @@ def open(self, name, mode="r", pwd=None, *, force_zip64=False): """ if mode not in {"r", "w"}: raise ValueError('open() requires mode "r" or "w"') - if pwd and not isinstance(pwd, bytes): - raise TypeError("pwd: expected bytes, got %s" % type(pwd).__name__) if pwd and (mode == "w"): raise ValueError("pwd is only supported for reading files") if not self.fp: @@ -1517,7 +1633,7 @@ def open(self, name, mode="r", pwd=None, *, force_zip64=False): elif mode == 'w': zinfo = ZipInfo(name) zinfo.compress_type = self.compression - zinfo._compresslevel = self.compresslevel + zinfo.compress_level = self.compresslevel else: # Get info object for name zinfo = self.getinfo(name) @@ -1545,39 +1661,54 @@ def open(self, name, mode="r", pwd=None, *, force_zip64=False): fname = zef_file.read(fheader[_FH_FILENAME_LENGTH]) if fheader[_FH_EXTRA_FIELD_LENGTH]: - zef_file.read(fheader[_FH_EXTRA_FIELD_LENGTH]) + zef_file.seek(fheader[_FH_EXTRA_FIELD_LENGTH], whence=1) - if zinfo.flag_bits & 0x20: + if zinfo.flag_bits & _MASK_COMPRESSED_PATCH: # Zip 2.7: compressed patched data raise NotImplementedError("compressed patched data (flag bit 5)") - if zinfo.flag_bits & 0x40: + if zinfo.flag_bits & _MASK_STRONG_ENCRYPTION: # strong encryption raise NotImplementedError("strong encryption (flag bit 6)") - if fheader[_FH_GENERAL_PURPOSE_FLAG_BITS] & 0x800: + if fheader[_FH_GENERAL_PURPOSE_FLAG_BITS] & _MASK_UTF_FILENAME: # UTF-8 filename fname_str = fname.decode("utf-8") else: - fname_str = fname.decode("cp437") + fname_str = fname.decode(self.metadata_encoding or "cp437") if fname_str != zinfo.orig_filename: raise BadZipFile( 'File name in directory %r and header %r differ.' % (zinfo.orig_filename, fname)) + if (zinfo._end_offset is not None and + zef_file.tell() + zinfo.compress_size > zinfo._end_offset): + if zinfo._end_offset == zinfo.header_offset: + import warnings + warnings.warn( + f"Overlapped entries: {zinfo.orig_filename!r} " + f"(possible zip bomb)", + skip_file_prefixes=(os.path.dirname(__file__),)) + else: + raise BadZipFile( + f"Overlapped entries: {zinfo.orig_filename!r} " + f"(possible zip bomb)") + # check for encrypted flag & handle password - is_encrypted = zinfo.flag_bits & 0x1 + is_encrypted = zinfo.flag_bits & _MASK_ENCRYPTED if is_encrypted: if not pwd: pwd = self.pwd + if pwd and not isinstance(pwd, bytes): + raise TypeError("pwd: expected bytes, got %s" % type(pwd).__name__) if not pwd: raise RuntimeError("File %r is encrypted, password " "required for extraction" % name) else: pwd = None - return ZipExtFile(zef_file, mode, zinfo, pwd, True) + return ZipExtFile(zef_file, mode + 'b', zinfo, pwd, True) except: zef_file.close() raise @@ -1600,16 +1731,17 @@ def _open_to_write(self, zinfo, force_zip64=False): zinfo.flag_bits = 0x00 if zinfo.compress_type == ZIP_LZMA: # Compressed data includes an end-of-stream (EOS) marker - zinfo.flag_bits |= 0x02 + zinfo.flag_bits |= _MASK_COMPRESS_OPTION_1 if not self._seekable: - zinfo.flag_bits |= 0x08 + zinfo.flag_bits |= _MASK_USE_DATA_DESCRIPTOR if not zinfo.external_attr: zinfo.external_attr = 0o600 << 16 # permissions: ?rw------- # Compressed size can be larger than uncompressed size - zip64 = self._allowZip64 and \ - (force_zip64 or zinfo.file_size * 1.05 > ZIP64_LIMIT) + zip64 = force_zip64 or (zinfo.file_size * 1.05 > ZIP64_LIMIT) + if not self._allowZip64 and zip64: + raise LargeZipFile("Filesize would require ZIP64 extensions") if self._seekable: self.fp.seek(self.start_dir) @@ -1627,7 +1759,8 @@ def extract(self, member, path=None, pwd=None): """Extract a member from the archive to the current working directory, using its full name. Its file information is extracted as accurately as possible. `member' may be a filename or a ZipInfo object. You can - specify a different directory using `path'. + specify a different directory using `path'. You can specify the + password to decrypt the file using 'pwd'. """ if path is None: path = os.getcwd() @@ -1640,7 +1773,8 @@ def extractall(self, path=None, members=None, pwd=None): """Extract all members from the archive to the current working directory. `path' specifies a different directory to extract to. `members' is optional and must be a subset of the list returned - by namelist(). + by namelist(). You can specify the password to decrypt all files + using 'pwd'. """ if members is None: members = self.namelist() @@ -1662,8 +1796,8 @@ def _sanitize_windows_name(cls, arcname, pathsep): table = str.maketrans(illegal, '_' * len(illegal)) cls._windows_illegal_name_trans_table = table arcname = arcname.translate(table) - # remove trailing dots - arcname = (x.rstrip('.') for x in arcname.split(pathsep)) + # remove trailing dots and spaces + arcname = (x.rstrip(' .') for x in arcname.split(pathsep)) # rejoin, removing empty parts. arcname = pathsep.join(x for x in arcname if x) return arcname @@ -1691,17 +1825,24 @@ def _extract_member(self, member, targetpath, pwd): # filter illegal characters on Windows arcname = self._sanitize_windows_name(arcname, os.path.sep) + if not arcname and not member.is_dir(): + raise ValueError("Empty filename.") + targetpath = os.path.join(targetpath, arcname) targetpath = os.path.normpath(targetpath) # Create all upper directories if necessary. upperdirs = os.path.dirname(targetpath) if upperdirs and not os.path.exists(upperdirs): - os.makedirs(upperdirs) + os.makedirs(upperdirs, exist_ok=True) if member.is_dir(): if not os.path.isdir(targetpath): - os.mkdir(targetpath) + try: + os.mkdir(targetpath) + except FileExistsError: + if not os.path.isdir(targetpath): + raise return targetpath with self.open(member, pwd=pwd) as source, \ @@ -1751,6 +1892,7 @@ def write(self, filename, arcname=None, if zinfo.is_dir(): zinfo.compress_size = 0 zinfo.CRC = 0 + self.mkdir(zinfo) else: if compress_type is not None: zinfo.compress_type = compress_type @@ -1758,27 +1900,10 @@ def write(self, filename, arcname=None, zinfo.compress_type = self.compression if compresslevel is not None: - zinfo._compresslevel = compresslevel + zinfo.compress_level = compresslevel else: - zinfo._compresslevel = self.compresslevel + zinfo.compress_level = self.compresslevel - if zinfo.is_dir(): - with self._lock: - if self._seekable: - self.fp.seek(self.start_dir) - zinfo.header_offset = self.fp.tell() # Start of header bytes - if zinfo.compress_type == ZIP_LZMA: - # Compressed data includes an end-of-stream (EOS) marker - zinfo.flag_bits |= 0x02 - - self._writecheck(zinfo) - self._didModify = True - - self.filelist.append(zinfo) - self.NameToInfo[zinfo.filename] = zinfo - self.fp.write(zinfo.FileHeader(False)) - self.start_dir = self.fp.tell() - else: with open(filename, "rb") as src, self.open(zinfo, 'w') as dest: shutil.copyfileobj(src, dest, 1024*8) @@ -1795,8 +1920,8 @@ def writestr(self, zinfo_or_arcname, data, zinfo = ZipInfo(filename=zinfo_or_arcname, date_time=time.localtime(time.time())[:6]) zinfo.compress_type = self.compression - zinfo._compresslevel = self.compresslevel - if zinfo.filename[-1] == '/': + zinfo.compress_level = self.compresslevel + if zinfo.filename.endswith('/'): zinfo.external_attr = 0o40775 << 16 # drwxrwxr-x zinfo.external_attr |= 0x10 # MS-DOS directory flag else: @@ -1816,13 +1941,48 @@ def writestr(self, zinfo_or_arcname, data, zinfo.compress_type = compress_type if compresslevel is not None: - zinfo._compresslevel = compresslevel + zinfo.compress_level = compresslevel zinfo.file_size = len(data) # Uncompressed size with self._lock: with self.open(zinfo, mode='w') as dest: dest.write(data) + def mkdir(self, zinfo_or_directory_name, mode=511): + """Creates a directory inside the zip archive.""" + if isinstance(zinfo_or_directory_name, ZipInfo): + zinfo = zinfo_or_directory_name + if not zinfo.is_dir(): + raise ValueError("The given ZipInfo does not describe a directory") + elif isinstance(zinfo_or_directory_name, str): + directory_name = zinfo_or_directory_name + if not directory_name.endswith("/"): + directory_name += "/" + zinfo = ZipInfo(directory_name) + zinfo.compress_size = 0 + zinfo.CRC = 0 + zinfo.external_attr = ((0o40000 | mode) & 0xFFFF) << 16 + zinfo.file_size = 0 + zinfo.external_attr |= 0x10 + else: + raise TypeError("Expected type str or ZipInfo") + + with self._lock: + if self._seekable: + self.fp.seek(self.start_dir) + zinfo.header_offset = self.fp.tell() # Start of header bytes + if zinfo.compress_type == ZIP_LZMA: + # Compressed data includes an end-of-stream (EOS) marker + zinfo.flag_bits |= _MASK_COMPRESS_OPTION_1 + + self._writecheck(zinfo) + self._didModify = True + + self.filelist.append(zinfo) + self.NameToInfo[zinfo.filename] = zinfo + self.fp.write(zinfo.FileHeader(False)) + self.start_dir = self.fp.tell() + def __del__(self): """Call the "close()" method in case the user forgot.""" self.close() @@ -1875,7 +2035,7 @@ def _write_end_record(self): min_version = 0 if extra: # Append a ZIP64 field to the extra's - extra_data = _strip_extra(extra_data, (1,)) + extra_data = _Extra.strip(extra_data, (1,)) extra_data = struct.pack( '>> list(_parents('b/d')) - ['b'] - >>> list(_parents('/b/d/')) - ['/b'] - >>> list(_parents('b/d/f/')) - ['b/d', 'b'] - >>> list(_parents('b')) - [] - >>> list(_parents('')) - [] - """ - return itertools.islice(_ancestry(path), 1, None) - - -def _ancestry(path): - """ - Given a path with elements separated by - posixpath.sep, generate all elements of that path - - >>> list(_ancestry('b/d')) - ['b/d', 'b'] - >>> list(_ancestry('/b/d/')) - ['/b/d', '/b'] - >>> list(_ancestry('b/d/f/')) - ['b/d/f', 'b/d', 'b'] - >>> list(_ancestry('b')) - ['b'] - >>> list(_ancestry('')) - [] - """ - path = path.rstrip(posixpath.sep) - while path and path != posixpath.sep: - yield path - path, tail = posixpath.split(path) - - -_dedupe = dict.fromkeys -"""Deduplicate an iterable in original order""" - - -def _difference(minuend, subtrahend): - """ - Return items in minuend not in subtrahend, retaining order - with O(1) lookup. - """ - return itertools.filterfalse(set(subtrahend).__contains__, minuend) - - -class CompleteDirs(ZipFile): - """ - A ZipFile subclass that ensures that implied directories - are always included in the namelist. - """ - - @staticmethod - def _implied_dirs(names): - parents = itertools.chain.from_iterable(map(_parents, names)) - as_dirs = (p + posixpath.sep for p in parents) - return _dedupe(_difference(as_dirs, names)) - - def namelist(self): - names = super(CompleteDirs, self).namelist() - return names + list(self._implied_dirs(names)) - - def _name_set(self): - return set(self.namelist()) - - def resolve_dir(self, name): - """ - If the name represents a directory, return that name - as a directory (with the trailing slash). - """ - names = self._name_set() - dirname = name + '/' - dir_match = name not in names and dirname in names - return dirname if dir_match else name - - @classmethod - def make(cls, source): - """ - Given a source (filename or zipfile), return an - appropriate CompleteDirs subclass. - """ - if isinstance(source, CompleteDirs): - return source - - if not isinstance(source, ZipFile): - return cls(source) - - # Only allow for FastLookup when supplied zipfile is read-only - if 'r' not in source.mode: - cls = CompleteDirs - - source.__class__ = cls - return source - - -class FastLookup(CompleteDirs): - """ - ZipFile subclass to ensure implicit - dirs exist and are resolved rapidly. - """ - - def namelist(self): - with contextlib.suppress(AttributeError): - return self.__names - self.__names = super(FastLookup, self).namelist() - return self.__names - - def _name_set(self): - with contextlib.suppress(AttributeError): - return self.__lookup - self.__lookup = super(FastLookup, self)._name_set() - return self.__lookup - - -class Path: - """ - A pathlib-compatible interface for zip files. - - Consider a zip file with this structure:: - - . - ├── a.txt - └── b - ├── c.txt - └── d - └── e.txt - - >>> data = io.BytesIO() - >>> zf = ZipFile(data, 'w') - >>> zf.writestr('a.txt', 'content of a') - >>> zf.writestr('b/c.txt', 'content of c') - >>> zf.writestr('b/d/e.txt', 'content of e') - >>> zf.filename = 'mem/abcde.zip' - - Path accepts the zipfile object itself or a filename - - >>> root = Path(zf) - - From there, several path operations are available. - - Directory iteration (including the zip file itself): - - >>> a, b = root.iterdir() - >>> a - Path('mem/abcde.zip', 'a.txt') - >>> b - Path('mem/abcde.zip', 'b/') - - name property: - - >>> b.name - 'b' - - join with divide operator: - - >>> c = b / 'c.txt' - >>> c - Path('mem/abcde.zip', 'b/c.txt') - >>> c.name - 'c.txt' - - Read text: - - >>> c.read_text() - 'content of c' - - existence: - - >>> c.exists() - True - >>> (b / 'missing.txt').exists() - False - - Coercion to string: - - >>> import os - >>> str(c).replace(os.sep, posixpath.sep) - 'mem/abcde.zip/b/c.txt' - - At the root, ``name``, ``filename``, and ``parent`` - resolve to the zipfile. Note these attributes are not - valid and will raise a ``ValueError`` if the zipfile - has no filename. - - >>> root.name - 'abcde.zip' - >>> str(root.filename).replace(os.sep, posixpath.sep) - 'mem/abcde.zip' - >>> str(root.parent) - 'mem' - """ - - __repr = "{self.__class__.__name__}({self.root.filename!r}, {self.at!r})" - - def __init__(self, root, at=""): - """ - Construct a Path from a ZipFile or filename. - - Note: When the source is an existing ZipFile object, - its type (__class__) will be mutated to a - specialized type. If the caller wishes to retain the - original type, the caller should either create a - separate ZipFile object or pass a filename. - """ - self.root = FastLookup.make(root) - self.at = at - - def open(self, mode='r', *args, pwd=None, **kwargs): - """ - Open this entry as text or binary following the semantics - of ``pathlib.Path.open()`` by passing arguments through - to io.TextIOWrapper(). - """ - if self.is_dir(): - raise IsADirectoryError(self) - zip_mode = mode[0] - if not self.exists() and zip_mode == 'r': - raise FileNotFoundError(self) - stream = self.root.open(self.at, zip_mode, pwd=pwd) - if 'b' in mode: - if args or kwargs: - raise ValueError("encoding args invalid for binary operation") - return stream - else: - kwargs["encoding"] = io.text_encoding(kwargs.get("encoding")) - return io.TextIOWrapper(stream, *args, **kwargs) - - @property - def name(self): - return pathlib.Path(self.at).name or self.filename.name - - @property - def filename(self): - return pathlib.Path(self.root.filename).joinpath(self.at) - - def read_text(self, *args, **kwargs): - kwargs["encoding"] = io.text_encoding(kwargs.get("encoding")) - with self.open('r', *args, **kwargs) as strm: - return strm.read() - - def read_bytes(self): - with self.open('rb') as strm: - return strm.read() - - def _is_child(self, path): - return posixpath.dirname(path.at.rstrip("/")) == self.at.rstrip("/") - - def _next(self, at): - return self.__class__(self.root, at) - - def is_dir(self): - return not self.at or self.at.endswith("/") - - def is_file(self): - return self.exists() and not self.is_dir() - - def exists(self): - return self.at in self.root._name_set() - - def iterdir(self): - if not self.is_dir(): - raise ValueError("Can't listdir a file") - subs = map(self._next, self.root.namelist()) - return filter(self._is_child, subs) - - def __str__(self): - return posixpath.join(self.root.filename, self.at) - - def __repr__(self): - return self.__repr.format(self=self) - - def joinpath(self, *other): - next = posixpath.join(self.at, *other) - return self._next(self.root.resolve_dir(next)) - - __truediv__ = joinpath - - @property - def parent(self): - if not self.at: - return self.filename.parent - parent_at = posixpath.dirname(self.at.rstrip('/')) - if parent_at: - parent_at += '/' - return self._next(parent_at) - - def main(args=None): import argparse @@ -2434,11 +2300,15 @@ def main(args=None): help='Create zipfile from sources') group.add_argument('-t', '--test', metavar='', help='Test if a zipfile is valid') + parser.add_argument('--metadata-encoding', metavar='', + help='Specify encoding of member names for -l, -e and -t') args = parser.parse_args(args) + encoding = args.metadata_encoding + if args.test is not None: src = args.test - with ZipFile(src, 'r') as zf: + with ZipFile(src, 'r', metadata_encoding=encoding) as zf: badfile = zf.testzip() if badfile: print("The following enclosed file is corrupted: {!r}".format(badfile)) @@ -2446,15 +2316,20 @@ def main(args=None): elif args.list is not None: src = args.list - with ZipFile(src, 'r') as zf: + with ZipFile(src, 'r', metadata_encoding=encoding) as zf: zf.printdir() elif args.extract is not None: src, curdir = args.extract - with ZipFile(src, 'r') as zf: + with ZipFile(src, 'r', metadata_encoding=encoding) as zf: zf.extractall(curdir) elif args.create is not None: + if encoding: + print("Non-conforming encodings not supported with -c.", + file=sys.stderr) + sys.exit(1) + zip_name = args.create.pop(0) files = args.create @@ -2479,5 +2354,9 @@ def addToZip(zf, path, zippath): addToZip(zf, path, zippath) -if __name__ == "__main__": - main() +from ._path import ( # noqa: E402 + Path, + + # used privately for tests + CompleteDirs, # noqa: F401 +) diff --git a/Lib/zipfile/__main__.py b/Lib/zipfile/__main__.py new file mode 100644 index 00000000000..868d99efc3c --- /dev/null +++ b/Lib/zipfile/__main__.py @@ -0,0 +1,4 @@ +from . import main + +if __name__ == "__main__": + main() diff --git a/Lib/zipfile/_path/__init__.py b/Lib/zipfile/_path/__init__.py new file mode 100644 index 00000000000..02f81171b4f --- /dev/null +++ b/Lib/zipfile/_path/__init__.py @@ -0,0 +1,452 @@ +""" +A Path-like interface for zipfiles. + +This codebase is shared between zipfile.Path in the stdlib +and zipp in PyPI. See +https://github.com/python/importlib_metadata/wiki/Development-Methodology +for more detail. +""" + +import contextlib +import io +import itertools +import pathlib +import posixpath +import re +import stat +import sys +import zipfile + +from .glob import Translator + +__all__ = ['Path'] + + +def _parents(path): + """ + Given a path with elements separated by + posixpath.sep, generate all parents of that path. + + >>> list(_parents('b/d')) + ['b'] + >>> list(_parents('/b/d/')) + ['/b'] + >>> list(_parents('b/d/f/')) + ['b/d', 'b'] + >>> list(_parents('b')) + [] + >>> list(_parents('')) + [] + """ + return itertools.islice(_ancestry(path), 1, None) + + +def _ancestry(path): + """ + Given a path with elements separated by + posixpath.sep, generate all elements of that path. + + >>> list(_ancestry('b/d')) + ['b/d', 'b'] + >>> list(_ancestry('/b/d/')) + ['/b/d', '/b'] + >>> list(_ancestry('b/d/f/')) + ['b/d/f', 'b/d', 'b'] + >>> list(_ancestry('b')) + ['b'] + >>> list(_ancestry('')) + [] + + Multiple separators are treated like a single. + + >>> list(_ancestry('//b//d///f//')) + ['//b//d///f', '//b//d', '//b'] + """ + path = path.rstrip(posixpath.sep) + while path.rstrip(posixpath.sep): + yield path + path, tail = posixpath.split(path) + + +_dedupe = dict.fromkeys +"""Deduplicate an iterable in original order""" + + +def _difference(minuend, subtrahend): + """ + Return items in minuend not in subtrahend, retaining order + with O(1) lookup. + """ + return itertools.filterfalse(set(subtrahend).__contains__, minuend) + + +class InitializedState: + """ + Mix-in to save the initialization state for pickling. + """ + + def __init__(self, *args, **kwargs): + self.__args = args + self.__kwargs = kwargs + super().__init__(*args, **kwargs) + + def __getstate__(self): + return self.__args, self.__kwargs + + def __setstate__(self, state): + args, kwargs = state + super().__init__(*args, **kwargs) + + +class CompleteDirs(InitializedState, zipfile.ZipFile): + """ + A ZipFile subclass that ensures that implied directories + are always included in the namelist. + + >>> list(CompleteDirs._implied_dirs(['foo/bar.txt', 'foo/bar/baz.txt'])) + ['foo/', 'foo/bar/'] + >>> list(CompleteDirs._implied_dirs(['foo/bar.txt', 'foo/bar/baz.txt', 'foo/bar/'])) + ['foo/'] + """ + + @staticmethod + def _implied_dirs(names): + parents = itertools.chain.from_iterable(map(_parents, names)) + as_dirs = (p + posixpath.sep for p in parents) + return _dedupe(_difference(as_dirs, names)) + + def namelist(self): + names = super().namelist() + return names + list(self._implied_dirs(names)) + + def _name_set(self): + return set(self.namelist()) + + def resolve_dir(self, name): + """ + If the name represents a directory, return that name + as a directory (with the trailing slash). + """ + names = self._name_set() + dirname = name + '/' + dir_match = name not in names and dirname in names + return dirname if dir_match else name + + def getinfo(self, name): + """ + Supplement getinfo for implied dirs. + """ + try: + return super().getinfo(name) + except KeyError: + if not name.endswith('/') or name not in self._name_set(): + raise + return zipfile.ZipInfo(filename=name) + + @classmethod + def make(cls, source): + """ + Given a source (filename or zipfile), return an + appropriate CompleteDirs subclass. + """ + if isinstance(source, CompleteDirs): + return source + + if not isinstance(source, zipfile.ZipFile): + return cls(source) + + # Only allow for FastLookup when supplied zipfile is read-only + if 'r' not in source.mode: + cls = CompleteDirs + + source.__class__ = cls + return source + + @classmethod + def inject(cls, zf: zipfile.ZipFile) -> zipfile.ZipFile: + """ + Given a writable zip file zf, inject directory entries for + any directories implied by the presence of children. + """ + for name in cls._implied_dirs(zf.namelist()): + zf.writestr(name, b"") + return zf + + +class FastLookup(CompleteDirs): + """ + ZipFile subclass to ensure implicit + dirs exist and are resolved rapidly. + """ + + def namelist(self): + with contextlib.suppress(AttributeError): + return self.__names + self.__names = super().namelist() + return self.__names + + def _name_set(self): + with contextlib.suppress(AttributeError): + return self.__lookup + self.__lookup = super()._name_set() + return self.__lookup + +def _extract_text_encoding(encoding=None, *args, **kwargs): + # compute stack level so that the caller of the caller sees any warning. + is_pypy = sys.implementation.name == 'pypy' + # PyPy no longer special cased after 7.3.19 (or maybe 7.3.18) + # See jaraco/zipp#143 + is_old_pypi = is_pypy and sys.pypy_version_info < (7, 3, 19) + stack_level = 3 + is_old_pypi + return io.text_encoding(encoding, stack_level), args, kwargs + + +class Path: + """ + A :class:`importlib.resources.abc.Traversable` interface for zip files. + + Implements many of the features users enjoy from + :class:`pathlib.Path`. + + Consider a zip file with this structure:: + + . + ├── a.txt + └── b + ├── c.txt + └── d + └── e.txt + + >>> data = io.BytesIO() + >>> zf = ZipFile(data, 'w') + >>> zf.writestr('a.txt', 'content of a') + >>> zf.writestr('b/c.txt', 'content of c') + >>> zf.writestr('b/d/e.txt', 'content of e') + >>> zf.filename = 'mem/abcde.zip' + + Path accepts the zipfile object itself or a filename + + >>> path = Path(zf) + + From there, several path operations are available. + + Directory iteration (including the zip file itself): + + >>> a, b = path.iterdir() + >>> a + Path('mem/abcde.zip', 'a.txt') + >>> b + Path('mem/abcde.zip', 'b/') + + name property: + + >>> b.name + 'b' + + join with divide operator: + + >>> c = b / 'c.txt' + >>> c + Path('mem/abcde.zip', 'b/c.txt') + >>> c.name + 'c.txt' + + Read text: + + >>> c.read_text(encoding='utf-8') + 'content of c' + + existence: + + >>> c.exists() + True + >>> (b / 'missing.txt').exists() + False + + Coercion to string: + + >>> import os + >>> str(c).replace(os.sep, posixpath.sep) + 'mem/abcde.zip/b/c.txt' + + At the root, ``name``, ``filename``, and ``parent`` + resolve to the zipfile. + + >>> str(path) + 'mem/abcde.zip/' + >>> path.name + 'abcde.zip' + >>> path.filename == pathlib.Path('mem/abcde.zip') + True + >>> str(path.parent) + 'mem' + + If the zipfile has no filename, such attributes are not + valid and accessing them will raise an Exception. + + >>> zf.filename = None + >>> path.name + Traceback (most recent call last): + ... + TypeError: ... + + >>> path.filename + Traceback (most recent call last): + ... + TypeError: ... + + >>> path.parent + Traceback (most recent call last): + ... + TypeError: ... + + # workaround python/cpython#106763 + >>> pass + """ + + __repr = "{self.__class__.__name__}({self.root.filename!r}, {self.at!r})" + + def __init__(self, root, at=""): + """ + Construct a Path from a ZipFile or filename. + + Note: When the source is an existing ZipFile object, + its type (__class__) will be mutated to a + specialized type. If the caller wishes to retain the + original type, the caller should either create a + separate ZipFile object or pass a filename. + """ + self.root = FastLookup.make(root) + self.at = at + + def __eq__(self, other): + """ + >>> Path(zipfile.ZipFile(io.BytesIO(), 'w')) == 'foo' + False + """ + if self.__class__ is not other.__class__: + return NotImplemented + return (self.root, self.at) == (other.root, other.at) + + def __hash__(self): + return hash((self.root, self.at)) + + def open(self, mode='r', *args, pwd=None, **kwargs): + """ + Open this entry as text or binary following the semantics + of ``pathlib.Path.open()`` by passing arguments through + to io.TextIOWrapper(). + """ + if self.is_dir(): + raise IsADirectoryError(self) + zip_mode = mode[0] + if zip_mode == 'r' and not self.exists(): + raise FileNotFoundError(self) + stream = self.root.open(self.at, zip_mode, pwd=pwd) + if 'b' in mode: + if args or kwargs: + raise ValueError("encoding args invalid for binary operation") + return stream + # Text mode: + encoding, args, kwargs = _extract_text_encoding(*args, **kwargs) + return io.TextIOWrapper(stream, encoding, *args, **kwargs) + + def _base(self): + return pathlib.PurePosixPath(self.at) if self.at else self.filename + + @property + def name(self): + return self._base().name + + @property + def suffix(self): + return self._base().suffix + + @property + def suffixes(self): + return self._base().suffixes + + @property + def stem(self): + return self._base().stem + + @property + def filename(self): + return pathlib.Path(self.root.filename).joinpath(self.at) + + def read_text(self, *args, **kwargs): + encoding, args, kwargs = _extract_text_encoding(*args, **kwargs) + with self.open('r', encoding, *args, **kwargs) as strm: + return strm.read() + + def read_bytes(self): + with self.open('rb') as strm: + return strm.read() + + def _is_child(self, path): + return posixpath.dirname(path.at.rstrip("/")) == self.at.rstrip("/") + + def _next(self, at): + return self.__class__(self.root, at) + + def is_dir(self): + return not self.at or self.at.endswith("/") + + def is_file(self): + return self.exists() and not self.is_dir() + + def exists(self): + return self.at in self.root._name_set() + + def iterdir(self): + if not self.is_dir(): + raise ValueError("Can't listdir a file") + subs = map(self._next, self.root.namelist()) + return filter(self._is_child, subs) + + def match(self, path_pattern): + return pathlib.PurePosixPath(self.at).match(path_pattern) + + def is_symlink(self): + """ + Return whether this path is a symlink. + """ + info = self.root.getinfo(self.at) + mode = info.external_attr >> 16 + return stat.S_ISLNK(mode) + + def glob(self, pattern): + if not pattern: + raise ValueError(f"Unacceptable pattern: {pattern!r}") + + prefix = re.escape(self.at) + tr = Translator(seps='/') + matches = re.compile(prefix + tr.translate(pattern)).fullmatch + return map(self._next, filter(matches, self.root.namelist())) + + def rglob(self, pattern): + return self.glob(f'**/{pattern}') + + def relative_to(self, other, *extra): + return posixpath.relpath(str(self), str(other.joinpath(*extra))) + + def __str__(self): + return posixpath.join(self.root.filename, self.at) + + def __repr__(self): + return self.__repr.format(self=self) + + def joinpath(self, *other): + next = posixpath.join(self.at, *other) + return self._next(self.root.resolve_dir(next)) + + __truediv__ = joinpath + + @property + def parent(self): + if not self.at: + return self.filename.parent + parent_at = posixpath.dirname(self.at.rstrip('/')) + if parent_at: + parent_at += '/' + return self._next(parent_at) diff --git a/Lib/zipfile/_path/glob.py b/Lib/zipfile/_path/glob.py new file mode 100644 index 00000000000..4ed74cc48d9 --- /dev/null +++ b/Lib/zipfile/_path/glob.py @@ -0,0 +1,113 @@ +import os +import re + +_default_seps = os.sep + str(os.altsep) * bool(os.altsep) + + +class Translator: + """ + >>> Translator('xyz') + Traceback (most recent call last): + ... + AssertionError: Invalid separators + + >>> Translator('') + Traceback (most recent call last): + ... + AssertionError: Invalid separators + """ + + seps: str + + def __init__(self, seps: str = _default_seps): + assert seps and set(seps) <= set(_default_seps), "Invalid separators" + self.seps = seps + + def translate(self, pattern): + """ + Given a glob pattern, produce a regex that matches it. + """ + return self.extend(self.match_dirs(self.translate_core(pattern))) + + def extend(self, pattern): + r""" + Extend regex for pattern-wide concerns. + + Apply '(?s:)' to create a non-matching group that + matches newlines (valid on Unix). + + Append '\Z' to imply fullmatch even when match is used. + """ + return rf'(?s:{pattern})\Z' + + def match_dirs(self, pattern): + """ + Ensure that zipfile.Path directory names are matched. + + zipfile.Path directory names always end in a slash. + """ + return rf'{pattern}[/]?' + + def translate_core(self, pattern): + r""" + Given a glob pattern, produce a regex that matches it. + + >>> t = Translator() + >>> t.translate_core('*.txt').replace('\\\\', '') + '[^/]*\\.txt' + >>> t.translate_core('a?txt') + 'a[^/]txt' + >>> t.translate_core('**/*').replace('\\\\', '') + '.*/[^/][^/]*' + """ + self.restrict_rglob(pattern) + return ''.join(map(self.replace, separate(self.star_not_empty(pattern)))) + + def replace(self, match): + """ + Perform the replacements for a match from :func:`separate`. + """ + return match.group('set') or ( + re.escape(match.group(0)) + .replace('\\*\\*', r'.*') + .replace('\\*', rf'[^{re.escape(self.seps)}]*') + .replace('\\?', r'[^/]') + ) + + def restrict_rglob(self, pattern): + """ + Raise ValueError if ** appears in anything but a full path segment. + + >>> Translator().translate('**foo') + Traceback (most recent call last): + ... + ValueError: ** must appear alone in a path segment + """ + seps_pattern = rf'[{re.escape(self.seps)}]+' + segments = re.split(seps_pattern, pattern) + if any('**' in segment and segment != '**' for segment in segments): + raise ValueError("** must appear alone in a path segment") + + def star_not_empty(self, pattern): + """ + Ensure that * will not match an empty segment. + """ + + def handle_segment(match): + segment = match.group(0) + return '?*' if segment == '*' else segment + + not_seps_pattern = rf'[^{re.escape(self.seps)}]+' + return re.sub(not_seps_pattern, handle_segment, pattern) + + +def separate(pattern): + """ + Separate out character sets to avoid translating their contents. + + >>> [m.group(0) for m in separate('*.txt')] + ['*.txt'] + >>> [m.group(0) for m in separate('a[?]txt')] + ['a', '[?]', 'txt'] + """ + return re.finditer(r'([^\[]+)|(?P[\[].*?[\]])|([\[][^\]]*$)', pattern) From dac236dac077d737298d7570d10b5833a8c64f62 Mon Sep 17 00:00:00 2001 From: "Jeong, YunWon" <69878+youknowone@users.noreply.github.com> Date: Tue, 30 Dec 2025 15:15:02 +0900 Subject: [PATCH 0003/1575] more no_std clippy (#6587) --- crates/common/src/os.rs | 2 +- crates/jit/tests/common.rs | 2 +- crates/sre_engine/benches/benches.rs | 2 +- crates/stdlib/src/scproxy.rs | 2 +- crates/stdlib/src/socket.rs | 32 +++++++++++++-------------- crates/stdlib/src/sqlite.rs | 6 +++-- crates/vm/src/function/builtin.rs | 2 +- crates/vm/src/stdlib/ctypes.rs | 2 +- crates/vm/src/stdlib/ctypes/array.rs | 4 ++-- crates/vm/src/stdlib/ctypes/simple.rs | 2 +- crates/vm/src/stdlib/os.rs | 10 ++++----- crates/vm/src/stdlib/posix.rs | 4 ++-- crates/vm/src/stdlib/thread.rs | 14 ++++++------ crates/wasm/src/browser_module.rs | 3 ++- crates/wasm/src/js_module.rs | 2 +- crates/wasm/src/lib.rs | 2 ++ crates/wasm/src/vm_class.rs | 8 +++---- 17 files changed, 50 insertions(+), 49 deletions(-) diff --git a/crates/common/src/os.rs b/crates/common/src/os.rs index 3e09a29210a..1ce25988d28 100644 --- a/crates/common/src/os.rs +++ b/crates/common/src/os.rs @@ -89,7 +89,7 @@ pub fn bytes_as_os_str(b: &[u8]) -> Result<&std::ffi::OsStr, Utf8Error> { #[cfg(not(unix))] pub fn bytes_as_os_str(b: &[u8]) -> Result<&std::ffi::OsStr, Utf8Error> { - Ok(std::str::from_utf8(b)?.as_ref()) + Ok(core::str::from_utf8(b)?.as_ref()) } #[cfg(unix)] diff --git a/crates/jit/tests/common.rs b/crates/jit/tests/common.rs index 5dafeaeb807..17c280ec3c9 100644 --- a/crates/jit/tests/common.rs +++ b/crates/jit/tests/common.rs @@ -1,9 +1,9 @@ +use core::ops::ControlFlow; use rustpython_compiler_core::bytecode::{ CodeObject, ConstantData, Instruction, OpArg, OpArgState, }; use rustpython_jit::{CompiledCode, JitType}; use std::collections::HashMap; -use std::ops::ControlFlow; #[derive(Debug, Clone)] pub struct Function { diff --git a/crates/sre_engine/benches/benches.rs b/crates/sre_engine/benches/benches.rs index 127f72e2747..9905a8db70f 100644 --- a/crates/sre_engine/benches/benches.rs +++ b/crates/sre_engine/benches/benches.rs @@ -15,7 +15,7 @@ impl Pattern { fn state_range<'a, S: StrDrive>( &self, string: S, - range: std::ops::Range, + range: core::ops::Range, ) -> (Request<'a, S>, State) { let req = Request::new(string, range.start, range.end, self.code, false); let state = State::default(); diff --git a/crates/stdlib/src/scproxy.rs b/crates/stdlib/src/scproxy.rs index 40267579029..b105dc4f2fb 100644 --- a/crates/stdlib/src/scproxy.rs +++ b/crates/stdlib/src/scproxy.rs @@ -91,7 +91,7 @@ mod _scproxy { .find(host_key) .and_then(|v| v.downcast::()) { - let h = std::borrow::Cow::::from(&host); + let h = alloc::borrow::Cow::::from(&host); let v = if let Some(port) = proxy_dict .find(port_key) .and_then(|v| v.downcast::()) diff --git a/crates/stdlib/src/socket.rs b/crates/stdlib/src/socket.rs index 71bc8e9f170..a37a4fd241d 100644 --- a/crates/stdlib/src/socket.rs +++ b/crates/stdlib/src/socket.rs @@ -1228,7 +1228,7 @@ mod _socket { .new_os_error("interface name too long".to_owned()) .into()); } - let cstr = std::ffi::CString::new(ifname) + let cstr = alloc::ffi::CString::new(ifname) .map_err(|_| vm.new_os_error("invalid interface name".to_owned()))?; let idx = unsafe { libc::if_nametoindex(cstr.as_ptr()) }; if idx == 0 { @@ -1238,7 +1238,7 @@ mod _socket { }; // Create sockaddr_can - let mut storage: libc::sockaddr_storage = unsafe { std::mem::zeroed() }; + let mut storage: libc::sockaddr_storage = unsafe { core::mem::zeroed() }; let can_addr = &mut storage as *mut libc::sockaddr_storage as *mut libc::sockaddr_can; unsafe { @@ -1294,7 +1294,7 @@ mod _socket { } // Create sockaddr_alg - let mut storage: libc::sockaddr_storage = unsafe { std::mem::zeroed() }; + let mut storage: libc::sockaddr_storage = unsafe { core::mem::zeroed() }; let alg_addr = &mut storage as *mut libc::sockaddr_storage as *mut libc::sockaddr_alg; unsafe { @@ -1936,7 +1936,7 @@ mod _socket { flags: OptionalArg, vm: &VirtualMachine, ) -> PyResult { - use std::mem::MaybeUninit; + use core::mem::MaybeUninit; if bufsize < 0 { return Err(vm.new_value_error("negative buffer size in recvmsg".to_owned())); @@ -1955,7 +1955,7 @@ mod _socket { // Allocate buffers let mut data_buf: Vec> = vec![MaybeUninit::uninit(); bufsize]; let mut anc_buf: Vec> = vec![MaybeUninit::uninit(); ancbufsize]; - let mut addr_storage: libc::sockaddr_storage = unsafe { std::mem::zeroed() }; + let mut addr_storage: libc::sockaddr_storage = unsafe { core::mem::zeroed() }; // Set up iovec let mut iov = [libc::iovec { @@ -1964,9 +1964,9 @@ mod _socket { }]; // Set up msghdr - let mut msg: libc::msghdr = unsafe { std::mem::zeroed() }; + let mut msg: libc::msghdr = unsafe { core::mem::zeroed() }; msg.msg_name = (&mut addr_storage as *mut libc::sockaddr_storage).cast(); - msg.msg_namelen = std::mem::size_of::() as libc::socklen_t; + msg.msg_namelen = core::mem::size_of::() as libc::socklen_t; msg.msg_iov = iov.as_mut_ptr(); msg.msg_iovlen = 1; if ancbufsize > 0 { @@ -1990,7 +1990,7 @@ mod _socket { // Build data bytes let data = unsafe { data_buf.set_len(n); - std::mem::transmute::>, Vec>(data_buf) + core::mem::transmute::>, Vec>(data_buf) }; // Build ancdata list @@ -1999,7 +1999,7 @@ mod _socket { // Build address tuple let address = if msg.msg_namelen > 0 { let storage: socket2::SockAddrStorage = - unsafe { std::mem::transmute(addr_storage) }; + unsafe { core::mem::transmute(addr_storage) }; let addr = unsafe { socket2::SockAddr::new(storage, msg.msg_namelen) }; get_addr_tuple(&addr, vm) } else { @@ -2034,7 +2034,7 @@ mod _socket { let available = ctrl_end as usize - data_ptr as usize; let data_len = data_len_from_cmsg.min(available); - let data = unsafe { std::slice::from_raw_parts(data_ptr, data_len) }; + let data = unsafe { core::slice::from_raw_parts(data_ptr, data_len) }; let tuple = vm.ctx.new_tuple(vec![ vm.ctx.new_int(cmsg_ref.cmsg_level).into(), @@ -2820,13 +2820,11 @@ mod _socket { let host = host_encoded.as_deref(); // Encode port using UTF-8 - let port: Option> = match opts.port.as_ref() { - Some(Either::A(s)) => { - Some(std::borrow::Cow::Borrowed(s.to_str().ok_or_else(|| { - vm.new_unicode_encode_error("surrogates not allowed".to_owned()) - })?)) - } - Some(Either::B(i)) => Some(std::borrow::Cow::Owned(i.to_string())), + let port: Option> = match opts.port.as_ref() { + Some(Either::A(s)) => Some(alloc::borrow::Cow::Borrowed(s.to_str().ok_or_else( + || vm.new_unicode_encode_error("surrogates not allowed".to_owned()), + )?)), + Some(Either::B(i)) => Some(alloc::borrow::Cow::Owned(i.to_string())), None => None, }; let port = port.as_ref().map(|p| p.as_ref()); diff --git a/crates/stdlib/src/sqlite.rs b/crates/stdlib/src/sqlite.rs index 3a82787cd8f..54c889ecb6b 100644 --- a/crates/stdlib/src/sqlite.rs +++ b/crates/stdlib/src/sqlite.rs @@ -3191,7 +3191,9 @@ mod _sqlite { } fn aggregate_context(self) -> *mut T { - unsafe { sqlite3_aggregate_context(self.ctx, std::mem::size_of::() as c_int).cast() } + unsafe { + sqlite3_aggregate_context(self.ctx, core::mem::size_of::() as c_int).cast() + } } fn result_exception(self, vm: &VirtualMachine, exc: PyBaseExceptionRef, msg: &str) { @@ -3297,7 +3299,7 @@ mod _sqlite { } else if nbytes < 0 { Err(vm.new_system_error("negative size with ptr")) } else { - Ok(unsafe { std::slice::from_raw_parts(p.cast(), nbytes as usize) }.to_vec()) + Ok(unsafe { core::slice::from_raw_parts(p.cast(), nbytes as usize) }.to_vec()) } } diff --git a/crates/vm/src/function/builtin.rs b/crates/vm/src/function/builtin.rs index 06fd6a44f54..444df64a8ef 100644 --- a/crates/vm/src/function/builtin.rs +++ b/crates/vm/src/function/builtin.rs @@ -218,7 +218,7 @@ into_py_native_fn_tuple!( #[cfg(test)] mod tests { use super::*; - use std::mem::size_of_val; + use core::mem::size_of_val; #[test] fn test_into_native_fn_noalloc() { diff --git a/crates/vm/src/stdlib/ctypes.rs b/crates/vm/src/stdlib/ctypes.rs index f3b6dd25aca..cbe3f5405b6 100644 --- a/crates/vm/src/stdlib/ctypes.rs +++ b/crates/vm/src/stdlib/ctypes.rs @@ -1180,7 +1180,7 @@ pub(crate) mod _ctypes { path: Option, vm: &VirtualMachine, ) -> PyResult { - use std::ffi::CString; + use alloc::ffi::CString; let path = match path { Some(p) if !vm.is_none(&p) => p, diff --git a/crates/vm/src/stdlib/ctypes/array.rs b/crates/vm/src/stdlib/ctypes/array.rs index 168da2bcc01..5c298d26a56 100644 --- a/crates/vm/src/stdlib/ctypes/array.rs +++ b/crates/vm/src/stdlib/ctypes/array.rs @@ -12,8 +12,8 @@ use crate::{ protocol::{BufferDescriptor, PyBuffer, PyNumberMethods, PySequenceMethods}, types::{AsBuffer, AsNumber, AsSequence, Constructor, Initializer}, }; +use alloc::borrow::Cow; use num_traits::{Signed, ToPrimitive}; -use std::borrow::Cow; /// Get itemsize from a PEP 3118 format string /// Extracts the type code (last char after endianness prefix) and returns its size @@ -890,7 +890,7 @@ impl PyCArray { // Python's from_buffer requires writable buffer, so this is safe. let ptr = slice.as_ptr() as *mut u8; let len = slice.len(); - let owned_slice = unsafe { std::slice::from_raw_parts_mut(ptr, len) }; + let owned_slice = unsafe { core::slice::from_raw_parts_mut(ptr, len) }; Self::write_element_to_buffer( owned_slice, final_offset, diff --git a/crates/vm/src/stdlib/ctypes/simple.rs b/crates/vm/src/stdlib/ctypes/simple.rs index 9835953812f..7bcfa203b02 100644 --- a/crates/vm/src/stdlib/ctypes/simple.rs +++ b/crates/vm/src/stdlib/ctypes/simple.rs @@ -1304,7 +1304,7 @@ impl PyCSimple { let ptr = slice.as_ptr() as *mut u8; let len = slice.len().min(buffer_bytes.len()); unsafe { - std::ptr::copy_nonoverlapping(buffer_bytes.as_ptr(), ptr, len); + core::ptr::copy_nonoverlapping(buffer_bytes.as_ptr(), ptr, len); } } Cow::Owned(vec) => { diff --git a/crates/vm/src/stdlib/os.rs b/crates/vm/src/stdlib/os.rs index add8763f5e1..fa1495fcdbf 100644 --- a/crates/vm/src/stdlib/os.rs +++ b/crates/vm/src/stdlib/os.rs @@ -1747,16 +1747,16 @@ pub(super) mod _os { // We extract raw bytes and interpret as a native-endian integer. // Note: The value may differ across architectures due to endianness. let f_fsid = { - let ptr = std::ptr::addr_of!(st.f_fsid) as *const u8; - let size = std::mem::size_of_val(&st.f_fsid); + let ptr = core::ptr::addr_of!(st.f_fsid) as *const u8; + let size = core::mem::size_of_val(&st.f_fsid); if size >= 8 { - let bytes = unsafe { std::slice::from_raw_parts(ptr, 8) }; + let bytes = unsafe { core::slice::from_raw_parts(ptr, 8) }; u64::from_ne_bytes([ bytes[0], bytes[1], bytes[2], bytes[3], bytes[4], bytes[5], bytes[6], bytes[7], ]) as libc::c_ulong } else if size >= 4 { - let bytes = unsafe { std::slice::from_raw_parts(ptr, 4) }; + let bytes = unsafe { core::slice::from_raw_parts(ptr, 4) }; u32::from_ne_bytes([bytes[0], bytes[1], bytes[2], bytes[3]]) as libc::c_ulong } else { 0 @@ -1784,7 +1784,7 @@ pub(super) mod _os { #[pyfunction] #[pyfunction(name = "fstatvfs")] fn statvfs(path: OsPathOrFd<'_>, vm: &VirtualMachine) -> PyResult { - let mut st: libc::statvfs = unsafe { std::mem::zeroed() }; + let mut st: libc::statvfs = unsafe { core::mem::zeroed() }; let ret = match &path { OsPathOrFd::Path(p) => { let cpath = p.clone().into_cstring(vm)?; diff --git a/crates/vm/src/stdlib/posix.rs b/crates/vm/src/stdlib/posix.rs index 4ec71eecbf2..65a659790b2 100644 --- a/crates/vm/src/stdlib/posix.rs +++ b/crates/vm/src/stdlib/posix.rs @@ -360,9 +360,9 @@ pub mod module { #[cfg(any(target_os = "macos", target_os = "ios"))] fn getgroups_impl() -> nix::Result> { + use core::ptr; use libc::{c_int, gid_t}; use nix::errno::Errno; - use std::ptr; let ret = unsafe { libc::getgroups(0, ptr::null_mut()) }; let mut groups = Vec::::with_capacity(Errno::result(ret)? as usize); let ret = unsafe { @@ -1825,7 +1825,7 @@ pub mod module { #[cfg(target_os = "macos")] #[pyfunction] fn _fcopyfile(in_fd: i32, out_fd: i32, flags: i32, vm: &VirtualMachine) -> PyResult<()> { - let ret = unsafe { fcopyfile(in_fd, out_fd, std::ptr::null_mut(), flags as u32) }; + let ret = unsafe { fcopyfile(in_fd, out_fd, core::ptr::null_mut(), flags as u32) }; if ret < 0 { Err(vm.new_last_errno_error()) } else { diff --git a/crates/vm/src/stdlib/thread.rs b/crates/vm/src/stdlib/thread.rs index 6ab754be094..eb881911659 100644 --- a/crates/vm/src/stdlib/thread.rs +++ b/crates/vm/src/stdlib/thread.rs @@ -192,7 +192,7 @@ pub(crate) mod _thread { #[derive(PyPayload)] struct RLock { mu: RawRMutex, - count: std::sync::atomic::AtomicUsize, + count: core::sync::atomic::AtomicUsize, } impl fmt::Debug for RLock { @@ -207,7 +207,7 @@ pub(crate) mod _thread { fn slot_new(cls: PyTypeRef, _args: FuncArgs, vm: &VirtualMachine) -> PyResult { Self { mu: RawRMutex::INIT, - count: std::sync::atomic::AtomicUsize::new(0), + count: core::sync::atomic::AtomicUsize::new(0), } .into_ref_with_type(vm, cls) .map(Into::into) @@ -220,7 +220,7 @@ pub(crate) mod _thread { let result = acquire_lock_impl!(&self.mu, args, vm)?; if result { self.count - .fetch_add(1, std::sync::atomic::Ordering::Relaxed); + .fetch_add(1, core::sync::atomic::Ordering::Relaxed); } Ok(result) } @@ -231,11 +231,11 @@ pub(crate) mod _thread { return Err(vm.new_runtime_error("release unlocked lock")); } debug_assert!( - self.count.load(std::sync::atomic::Ordering::Relaxed) > 0, + self.count.load(core::sync::atomic::Ordering::Relaxed) > 0, "RLock count underflow" ); self.count - .fetch_sub(1, std::sync::atomic::Ordering::Relaxed); + .fetch_sub(1, core::sync::atomic::Ordering::Relaxed); unsafe { self.mu.unlock() }; Ok(()) } @@ -247,7 +247,7 @@ pub(crate) mod _thread { self.mu.unlock(); }; } - self.count.store(0, std::sync::atomic::Ordering::Relaxed); + self.count.store(0, core::sync::atomic::Ordering::Relaxed); let new_mut = RawRMutex::INIT; let old_mutex: AtomicCell<&RawRMutex> = AtomicCell::new(&self.mu); @@ -264,7 +264,7 @@ pub(crate) mod _thread { #[pymethod] fn _recursion_count(&self) -> usize { if self.mu.is_owned_by_current_thread() { - self.count.load(std::sync::atomic::Ordering::Relaxed) + self.count.load(core::sync::atomic::Ordering::Relaxed) } else { 0 } diff --git a/crates/wasm/src/browser_module.rs b/crates/wasm/src/browser_module.rs index 0b978a4b563..9b6219cab6f 100644 --- a/crates/wasm/src/browser_module.rs +++ b/crates/wasm/src/browser_module.rs @@ -117,7 +117,8 @@ mod _browser { #[pyfunction] fn request_animation_frame(func: ArgCallable, vm: &VirtualMachine) -> PyResult { - use std::{cell::RefCell, rc::Rc}; + use alloc::rc::Rc; + use core::cell::RefCell; // this basic setup for request_animation_frame taken from: // https://rustwasm.github.io/wasm-bindgen/examples/request-animation-frame.html diff --git a/crates/wasm/src/js_module.rs b/crates/wasm/src/js_module.rs index d4f623da9f4..11d23bdf6d8 100644 --- a/crates/wasm/src/js_module.rs +++ b/crates/wasm/src/js_module.rs @@ -8,6 +8,7 @@ mod _js { vm_class::{WASMVirtualMachine, stored_vm_from_wasm}, weak_vm, }; + use core::{cell, fmt, future}; use js_sys::{Array, Object, Promise, Reflect}; use rustpython_vm::{ Py, PyObjectRef, PyPayload, PyRef, PyResult, TryFromObject, VirtualMachine, @@ -17,7 +18,6 @@ mod _js { protocol::PyIterReturn, types::{IterNext, Representable, SelfIter}, }; - use std::{cell, fmt, future}; use wasm_bindgen::{JsCast, closure::Closure, prelude::*}; use wasm_bindgen_futures::{JsFuture, future_to_promise}; diff --git a/crates/wasm/src/lib.rs b/crates/wasm/src/lib.rs index 8d1d19ddae1..64595fd63ab 100644 --- a/crates/wasm/src/lib.rs +++ b/crates/wasm/src/lib.rs @@ -1,3 +1,5 @@ +extern crate alloc; + pub mod browser_module; pub mod convert; pub mod js_module; diff --git a/crates/wasm/src/vm_class.rs b/crates/wasm/src/vm_class.rs index ad5df8dab96..0339a47821e 100644 --- a/crates/wasm/src/vm_class.rs +++ b/crates/wasm/src/vm_class.rs @@ -3,6 +3,8 @@ use crate::{ convert::{self, PyResultExt}, js_module, wasm_builtins, }; +use alloc::rc::{Rc, Weak}; +use core::cell::RefCell; use js_sys::{Object, TypeError}; use rustpython_vm::{ Interpreter, PyObjectRef, PyPayload, PyRef, PyResult, Settings, VirtualMachine, @@ -10,11 +12,7 @@ use rustpython_vm::{ compiler::Mode, scope::Scope, }; -use std::{ - cell::RefCell, - collections::HashMap, - rc::{Rc, Weak}, -}; +use std::collections::HashMap; use wasm_bindgen::prelude::*; pub(crate) struct StoredVirtualMachine { From 6ed0b4f0acfc3ca2c7a46cdf54ada7281d1821e1 Mon Sep 17 00:00:00 2001 From: "Jeong, YunWon" <69878+youknowone@users.noreply.github.com> Date: Tue, 30 Dec 2025 15:15:16 +0900 Subject: [PATCH 0004/1575] Add PythonFinallizationError and upgrade multiprocessing (#6592) * PythonFinallizationError * fix posixprocess * upgrade multiprocessing from 3.13.11 --- Lib/multiprocessing/connection.py | 2 +- Lib/multiprocessing/resource_tracker.py | 253 ++++++++++++++++-------- Lib/multiprocessing/util.py | 81 +++++++- crates/stdlib/src/posixsubprocess.rs | 12 ++ crates/vm/src/exceptions.rs | 13 ++ crates/vm/src/vm/vm_new.rs | 1 + 6 files changed, 283 insertions(+), 79 deletions(-) diff --git a/Lib/multiprocessing/connection.py b/Lib/multiprocessing/connection.py index 8caddd204d7..abd88adf76e 100644 --- a/Lib/multiprocessing/connection.py +++ b/Lib/multiprocessing/connection.py @@ -74,7 +74,7 @@ def arbitrary_address(family): if family == 'AF_INET': return ('localhost', 0) elif family == 'AF_UNIX': - return tempfile.mktemp(prefix='listener-', dir=util.get_temp_dir()) + return tempfile.mktemp(prefix='sock-', dir=util.get_temp_dir()) elif family == 'AF_PIPE': return tempfile.mktemp(prefix=r'\\.\pipe\pyc-%d-%d-' % (os.getpid(), next(_mmap_counter)), dir="") diff --git a/Lib/multiprocessing/resource_tracker.py b/Lib/multiprocessing/resource_tracker.py index 05633ac21a2..22e3bbcf21b 100644 --- a/Lib/multiprocessing/resource_tracker.py +++ b/Lib/multiprocessing/resource_tracker.py @@ -15,11 +15,15 @@ # this resource tracker process, "killall python" would probably leave unlinked # resources. +import base64 import os import signal import sys import threading import warnings +from collections import deque + +import json from . import spawn from . import util @@ -66,6 +70,14 @@ def __init__(self): self._fd = None self._pid = None self._exitcode = None + self._reentrant_messages = deque() + + # True to use colon-separated lines, rather than JSON lines, + # for internal communication. (Mainly for testing). + # Filenames not supported by the simple format will always be sent + # using JSON. + # The reader should understand all formats. + self._use_simple_format = True def _reentrant_call_error(self): # gh-109629: this happens if an explicit call to the ResourceTracker @@ -102,7 +114,7 @@ def _stop_locked( # This shouldn't happen (it might when called by a finalizer) # so we check for it anyway. if self._lock._recursion_count() > 1: - return self._reentrant_call_error() + raise self._reentrant_call_error() if self._fd is None: # not running return @@ -113,7 +125,12 @@ def _stop_locked( close(self._fd) self._fd = None - _, status = waitpid(self._pid, 0) + try: + _, status = waitpid(self._pid, 0) + except ChildProcessError: + self._pid = None + self._exitcode = None + return self._pid = None @@ -132,76 +149,119 @@ def ensure_running(self): This can be run from any process. Usually a child process will use the resource created by its parent.''' + return self._ensure_running_and_write() + + def _teardown_dead_process(self): + os.close(self._fd) + + # Clean-up to avoid dangling processes. + try: + # _pid can be None if this process is a child from another + # python process, which has started the resource_tracker. + if self._pid is not None: + os.waitpid(self._pid, 0) + except ChildProcessError: + # The resource_tracker has already been terminated. + pass + self._fd = None + self._pid = None + self._exitcode = None + + warnings.warn('resource_tracker: process died unexpectedly, ' + 'relaunching. Some resources might leak.') + + def _launch(self): + fds_to_pass = [] + try: + fds_to_pass.append(sys.stderr.fileno()) + except Exception: + pass + r, w = os.pipe() + try: + fds_to_pass.append(r) + # process will out live us, so no need to wait on pid + exe = spawn.get_executable() + args = [ + exe, + *util._args_from_interpreter_flags(), + '-c', + f'from multiprocessing.resource_tracker import main;main({r})', + ] + # bpo-33613: Register a signal mask that will block the signals. + # This signal mask will be inherited by the child that is going + # to be spawned and will protect the child from a race condition + # that can make the child die before it registers signal handlers + # for SIGINT and SIGTERM. The mask is unregistered after spawning + # the child. + prev_sigmask = None + try: + if _HAVE_SIGMASK: + prev_sigmask = signal.pthread_sigmask(signal.SIG_BLOCK, _IGNORED_SIGNALS) + pid = util.spawnv_passfds(exe, args, fds_to_pass) + finally: + if prev_sigmask is not None: + signal.pthread_sigmask(signal.SIG_SETMASK, prev_sigmask) + except: + os.close(w) + raise + else: + self._fd = w + self._pid = pid + finally: + os.close(r) + + def _make_probe_message(self): + """Return a probe message.""" + if self._use_simple_format: + return b'PROBE:0:noop\n' + return ( + json.dumps( + {"cmd": "PROBE", "rtype": "noop"}, + ensure_ascii=True, + separators=(",", ":"), + ) + + "\n" + ).encode("ascii") + + def _ensure_running_and_write(self, msg=None): with self._lock: if self._lock._recursion_count() > 1: # The code below is certainly not reentrant-safe, so bail out - return self._reentrant_call_error() + if msg is None: + raise self._reentrant_call_error() + return self._reentrant_messages.append(msg) + if self._fd is not None: # resource tracker was launched before, is it still running? - if self._check_alive(): - # => still alive - return - # => dead, launch it again - os.close(self._fd) - - # Clean-up to avoid dangling processes. + if msg is None: + to_send = self._make_probe_message() + else: + to_send = msg try: - # _pid can be None if this process is a child from another - # python process, which has started the resource_tracker. - if self._pid is not None: - os.waitpid(self._pid, 0) - except ChildProcessError: - # The resource_tracker has already been terminated. - pass - self._fd = None - self._pid = None - self._exitcode = None + self._write(to_send) + except OSError: + self._teardown_dead_process() + self._launch() - warnings.warn('resource_tracker: process died unexpectedly, ' - 'relaunching. Some resources might leak.') + msg = None # message was sent in probe + else: + self._launch() - fds_to_pass = [] - try: - fds_to_pass.append(sys.stderr.fileno()) - except Exception: - pass - cmd = 'from multiprocessing.resource_tracker import main;main(%d)' - r, w = os.pipe() + while True: try: - fds_to_pass.append(r) - # process will out live us, so no need to wait on pid - exe = spawn.get_executable() - args = [exe] + util._args_from_interpreter_flags() - args += ['-c', cmd % r] - # bpo-33613: Register a signal mask that will block the signals. - # This signal mask will be inherited by the child that is going - # to be spawned and will protect the child from a race condition - # that can make the child die before it registers signal handlers - # for SIGINT and SIGTERM. The mask is unregistered after spawning - # the child. - prev_sigmask = None - try: - if _HAVE_SIGMASK: - prev_sigmask = signal.pthread_sigmask(signal.SIG_BLOCK, _IGNORED_SIGNALS) - pid = util.spawnv_passfds(exe, args, fds_to_pass) - finally: - if prev_sigmask is not None: - signal.pthread_sigmask(signal.SIG_SETMASK, prev_sigmask) - except: - os.close(w) - raise - else: - self._fd = w - self._pid = pid - finally: - os.close(r) + reentrant_msg = self._reentrant_messages.popleft() + except IndexError: + break + self._write(reentrant_msg) + if msg is not None: + self._write(msg) def _check_alive(self): '''Check that the pipe has not been closed by sending a probe.''' try: # We cannot use send here as it calls ensure_running, creating # a cycle. - os.write(self._fd, b'PROBE:0:noop\n') + os.write(self._fd, self._make_probe_message()) except OSError: return False else: @@ -215,27 +275,42 @@ def unregister(self, name, rtype): '''Unregister name of resource with resource tracker.''' self._send('UNREGISTER', name, rtype) - def _send(self, cmd, name, rtype): - try: - self.ensure_running() - except ReentrantCallError: - # The code below might or might not work, depending on whether - # the resource tracker was already running and still alive. - # Better warn the user. - # (XXX is warnings.warn itself reentrant-safe? :-) - warnings.warn( - f"ResourceTracker called reentrantly for resource cleanup, " - f"which is unsupported. " - f"The {rtype} object {name!r} might leak.") - msg = '{0}:{1}:{2}\n'.format(cmd, name, rtype).encode('ascii') - if len(msg) > 512: - # posix guarantees that writes to a pipe of less than PIPE_BUF - # bytes are atomic, and that PIPE_BUF >= 512 - raise ValueError('msg too long') + def _write(self, msg): nbytes = os.write(self._fd, msg) - assert nbytes == len(msg), "nbytes {0:n} but len(msg) {1:n}".format( - nbytes, len(msg)) + assert nbytes == len(msg), f"{nbytes=} != {len(msg)=}" + def _send(self, cmd, name, rtype): + if self._use_simple_format and '\n' not in name: + msg = f"{cmd}:{name}:{rtype}\n".encode("ascii") + if len(msg) > 512: + # posix guarantees that writes to a pipe of less than PIPE_BUF + # bytes are atomic, and that PIPE_BUF >= 512 + raise ValueError('msg too long') + self._ensure_running_and_write(msg) + return + + # POSIX guarantees that writes to a pipe of less than PIPE_BUF (512 on Linux) + # bytes are atomic. Therefore, we want the message to be shorter than 512 bytes. + # POSIX shm_open() and sem_open() require the name, including its leading slash, + # to be at most NAME_MAX bytes (255 on Linux) + # With json.dump(..., ensure_ascii=True) every non-ASCII byte becomes a 6-char + # escape like \uDC80. + # As we want the overall message to be kept atomic and therefore smaller than 512, + # we encode encode the raw name bytes with URL-safe Base64 - so a 255 long name + # will not exceed 340 bytes. + b = name.encode('utf-8', 'surrogateescape') + if len(b) > 255: + raise ValueError('shared memory name too long (max 255 bytes)') + b64 = base64.urlsafe_b64encode(b).decode('ascii') + + payload = {"cmd": cmd, "rtype": rtype, "base64_name": b64} + msg = (json.dumps(payload, ensure_ascii=True, separators=(",", ":")) + "\n").encode("ascii") + + # The entire JSON message is guaranteed < PIPE_BUF (512 bytes) by construction. + assert len(msg) <= 512, f"internal error: message too long ({len(msg)} bytes)" + assert msg.startswith(b'{') + + self._ensure_running_and_write(msg) _resource_tracker = ResourceTracker() ensure_running = _resource_tracker.ensure_running @@ -244,6 +319,30 @@ def _send(self, cmd, name, rtype): getfd = _resource_tracker.getfd +def _decode_message(line): + if line.startswith(b'{'): + try: + obj = json.loads(line.decode('ascii')) + except Exception as e: + raise ValueError("malformed resource_tracker message: %r" % (line,)) from e + + cmd = obj["cmd"] + rtype = obj["rtype"] + b64 = obj.get("base64_name", "") + + if not isinstance(cmd, str) or not isinstance(rtype, str) or not isinstance(b64, str): + raise ValueError("malformed resource_tracker fields: %r" % (obj,)) + + try: + name = base64.urlsafe_b64decode(b64).decode('utf-8', 'surrogateescape') + except ValueError as e: + raise ValueError("malformed resource_tracker base64_name: %r" % (b64,)) from e + else: + cmd, rest = line.strip().decode('ascii').split(':', maxsplit=1) + name, rtype = rest.rsplit(':', maxsplit=1) + return cmd, rtype, name + + def main(fd): '''Run resource tracker.''' # protect the process from ^C and "killall python" etc @@ -266,7 +365,7 @@ def main(fd): with open(fd, 'rb') as f: for line in f: try: - cmd, name, rtype = line.strip().decode('ascii').split(':') + cmd, rtype, name = _decode_message(line) cleanup_func = _CLEANUP_FUNCS.get(rtype, None) if cleanup_func is None: raise ValueError( diff --git a/Lib/multiprocessing/util.py b/Lib/multiprocessing/util.py index 75dde02d88c..b8bfea045df 100644 --- a/Lib/multiprocessing/util.py +++ b/Lib/multiprocessing/util.py @@ -34,6 +34,7 @@ DEBUG = 10 INFO = 20 SUBWARNING = 25 +WARNING = 30 LOGGER_NAME = 'multiprocessing' DEFAULT_LOGGING_FORMAT = '[%(levelname)s/%(processName)s] %(message)s' @@ -53,6 +54,10 @@ def info(msg, *args): if _logger: _logger.log(INFO, msg, *args, stacklevel=2) +def _warn(msg, *args): + if _logger: + _logger.log(WARNING, msg, *args, stacklevel=2) + def sub_warning(msg, *args): if _logger: _logger.log(SUBWARNING, msg, *args, stacklevel=2) @@ -121,6 +126,23 @@ def is_abstract_socket_namespace(address): # Function returning a temp directory which will be removed on exit # +# Maximum length of a NULL-terminated [1] socket file path is usually +# between 92 and 108 [2], but Linux is known to use a size of 108 [3]. +# BSD-based systems usually use a size of 104 or 108 and Windows does +# not create AF_UNIX sockets. +# +# [1]: https://github.com/python/cpython/issues/140734 +# [2]: https://pubs.opengroup.org/onlinepubs/9799919799/basedefs/sys_un.h.html +# [3]: https://man7.org/linux/man-pages/man7/unix.7.html + +if sys.platform == 'linux': + _SUN_PATH_MAX = 108 +elif sys.platform.startswith(('openbsd', 'freebsd')): + _SUN_PATH_MAX = 104 +else: + # On Windows platforms, we do not create AF_UNIX sockets. + _SUN_PATH_MAX = None if os.name == 'nt' else 92 + def _remove_temp_dir(rmtree, tempdir): rmtree(tempdir) @@ -130,12 +152,69 @@ def _remove_temp_dir(rmtree, tempdir): if current_process is not None: current_process._config['tempdir'] = None +def _get_base_temp_dir(tempfile): + """Get a temporary directory where socket files will be created. + + To prevent additional imports, pass a pre-imported 'tempfile' module. + """ + if os.name == 'nt': + return None + # Most of the time, the default temporary directory is /tmp. Thus, + # listener sockets files "$TMPDIR/pymp-XXXXXXXX/sock-XXXXXXXX" do + # not have a path length exceeding SUN_PATH_MAX. + # + # If users specify their own temporary directory, we may be unable + # to create those files. Therefore, we fall back to the system-wide + # temporary directory /tmp, assumed to exist on POSIX systems. + # + # See https://github.com/python/cpython/issues/132124. + base_tempdir = tempfile.gettempdir() + # Files created in a temporary directory are suffixed by a string + # generated by tempfile._RandomNameSequence, which, by design, + # is 8 characters long. + # + # Thus, the socket file path length (without NULL terminator) will be: + # + # len(base_tempdir + '/pymp-XXXXXXXX' + '/sock-XXXXXXXX') + sun_path_len = len(base_tempdir) + 14 + 14 + # Strict inequality to account for the NULL terminator. + # See https://github.com/python/cpython/issues/140734. + if sun_path_len < _SUN_PATH_MAX: + return base_tempdir + # Fallback to the default system-wide temporary directory. + # This ignores user-defined environment variables. + # + # On POSIX systems, /tmp MUST be writable by any application [1]. + # We however emit a warning if this is not the case to prevent + # obscure errors later in the execution. + # + # On some legacy systems, /var/tmp and /usr/tmp can be present + # and will be used instead. + # + # [1]: https://refspecs.linuxfoundation.org/FHS_3.0/fhs/ch03s18.html + dirlist = ['/tmp', '/var/tmp', '/usr/tmp'] + try: + base_system_tempdir = tempfile._get_default_tempdir(dirlist) + except FileNotFoundError: + _warn("Process-wide temporary directory %s will not be usable for " + "creating socket files and no usable system-wide temporary " + "directory was found in %s", base_tempdir, dirlist) + # At this point, the system-wide temporary directory is not usable + # but we may assume that the user-defined one is, even if we will + # not be able to write socket files out there. + return base_tempdir + _warn("Ignoring user-defined temporary directory: %s", base_tempdir) + # at most max(map(len, dirlist)) + 14 + 14 = 36 characters + assert len(base_system_tempdir) + 14 + 14 < _SUN_PATH_MAX + return base_system_tempdir + def get_temp_dir(): # get name of a temp directory which will be automatically cleaned up tempdir = process.current_process()._config.get('tempdir') if tempdir is None: import shutil, tempfile - tempdir = tempfile.mkdtemp(prefix='pymp-') + base_tempdir = _get_base_temp_dir(tempfile) + tempdir = tempfile.mkdtemp(prefix='pymp-', dir=base_tempdir) info('created temp directory %s', tempdir) # keep a strong reference to shutil.rmtree(), since the finalizer # can be called late during Python shutdown diff --git a/crates/stdlib/src/posixsubprocess.rs b/crates/stdlib/src/posixsubprocess.rs index 1ebf6619c24..a6badc081d2 100644 --- a/crates/stdlib/src/posixsubprocess.rs +++ b/crates/stdlib/src/posixsubprocess.rs @@ -33,6 +33,18 @@ mod _posixsubprocess { #[pyfunction] fn fork_exec(args: ForkExecArgs<'_>, vm: &VirtualMachine) -> PyResult { + // Check for interpreter shutdown when preexec_fn is used + if args.preexec_fn.is_some() + && vm + .state + .finalizing + .load(std::sync::atomic::Ordering::Acquire) + { + return Err(vm.new_python_finalization_error( + "preexec_fn not supported at interpreter shutdown".to_owned(), + )); + } + let extra_groups = args .groups_list .as_ref() diff --git a/crates/vm/src/exceptions.rs b/crates/vm/src/exceptions.rs index b1a654d58c4..5d0c1c6f3ed 100644 --- a/crates/vm/src/exceptions.rs +++ b/crates/vm/src/exceptions.rs @@ -493,6 +493,7 @@ pub struct ExceptionZoo { pub runtime_error: &'static Py, pub not_implemented_error: &'static Py, pub recursion_error: &'static Py, + pub python_finalization_error: &'static Py, pub syntax_error: &'static Py, pub incomplete_input_error: &'static Py, pub indentation_error: &'static Py, @@ -804,6 +805,7 @@ impl ExceptionZoo { let runtime_error = PyRuntimeError::init_builtin_type(); let not_implemented_error = PyNotImplementedError::init_builtin_type(); let recursion_error = PyRecursionError::init_builtin_type(); + let python_finalization_error = PyPythonFinalizationError::init_builtin_type(); let syntax_error = PySyntaxError::init_builtin_type(); let incomplete_input_error = PyIncompleteInputError::init_builtin_type(); @@ -879,6 +881,7 @@ impl ExceptionZoo { runtime_error, not_implemented_error, recursion_error, + python_finalization_error, syntax_error, incomplete_input_error, indentation_error, @@ -992,6 +995,11 @@ impl ExceptionZoo { extend_exception!(PyRuntimeError, ctx, excs.runtime_error); extend_exception!(PyNotImplementedError, ctx, excs.not_implemented_error); extend_exception!(PyRecursionError, ctx, excs.recursion_error); + extend_exception!( + PyPythonFinalizationError, + ctx, + excs.python_finalization_error + ); extend_exception!(PySyntaxError, ctx, excs.syntax_error, { "msg" => ctx.new_static_getset( @@ -2111,6 +2119,11 @@ pub(super) mod types { #[repr(transparent)] pub struct PyRecursionError(PyRuntimeError); + #[pyexception(name, base = PyRuntimeError, ctx = "python_finalization_error", impl)] + #[derive(Debug)] + #[repr(transparent)] + pub struct PyPythonFinalizationError(PyRuntimeError); + #[pyexception(name, base = PyException, ctx = "syntax_error")] #[derive(Debug)] #[repr(transparent)] diff --git a/crates/vm/src/vm/vm_new.rs b/crates/vm/src/vm/vm_new.rs index ba09c8ecf69..4f25d6e2036 100644 --- a/crates/vm/src/vm/vm_new.rs +++ b/crates/vm/src/vm/vm_new.rs @@ -600,5 +600,6 @@ impl VirtualMachine { define_exception_fn!(fn new_zero_division_error, zero_division_error, ZeroDivisionError); define_exception_fn!(fn new_overflow_error, overflow_error, OverflowError); define_exception_fn!(fn new_runtime_error, runtime_error, RuntimeError); + define_exception_fn!(fn new_python_finalization_error, python_finalization_error, PythonFinalizationError); define_exception_fn!(fn new_memory_error, memory_error, MemoryError); } From 6bd1d90b6a8f010ddac7a047b3023cd93e151a9b Mon Sep 17 00:00:00 2001 From: Terry Tianlin Luan <44815845+terryluan12@users.noreply.github.com> Date: Tue, 30 Dec 2025 02:35:58 -0500 Subject: [PATCH 0005/1575] Updated bdb.py + test_bdb.py (#6593) * Updated bdb.py + test_bdb.py * Double checked test_bdb.py with the script --- Lib/bdb.py | 148 +++++++++++++++++++++++++++++++++---------- Lib/test/test_bdb.py | 95 +++++++++++++++++---------- 2 files changed, 177 insertions(+), 66 deletions(-) diff --git a/Lib/bdb.py b/Lib/bdb.py index 0f3eec653ba..f256b56daaa 100644 --- a/Lib/bdb.py +++ b/Lib/bdb.py @@ -3,6 +3,7 @@ import fnmatch import sys import os +from contextlib import contextmanager from inspect import CO_GENERATOR, CO_COROUTINE, CO_ASYNC_GENERATOR __all__ = ["BdbQuit", "Bdb", "Breakpoint"] @@ -32,7 +33,12 @@ def __init__(self, skip=None): self.skip = set(skip) if skip else None self.breaks = {} self.fncache = {} + self.frame_trace_lines_opcodes = {} self.frame_returning = None + self.trace_opcodes = False + self.enterframe = None + self.cmdframe = None + self.cmdlineno = None self._load_breaks() @@ -60,6 +66,12 @@ def reset(self): self.botframe = None self._set_stopinfo(None, None) + @contextmanager + def set_enterframe(self, frame): + self.enterframe = frame + yield + self.enterframe = None + def trace_dispatch(self, frame, event, arg): """Dispatch a trace function for debugged frames based on the event. @@ -84,24 +96,28 @@ def trace_dispatch(self, frame, event, arg): The arg parameter depends on the previous event. """ - if self.quitting: - return # None - if event == 'line': - return self.dispatch_line(frame) - if event == 'call': - return self.dispatch_call(frame, arg) - if event == 'return': - return self.dispatch_return(frame, arg) - if event == 'exception': - return self.dispatch_exception(frame, arg) - if event == 'c_call': - return self.trace_dispatch - if event == 'c_exception': - return self.trace_dispatch - if event == 'c_return': + + with self.set_enterframe(frame): + if self.quitting: + return # None + if event == 'line': + return self.dispatch_line(frame) + if event == 'call': + return self.dispatch_call(frame, arg) + if event == 'return': + return self.dispatch_return(frame, arg) + if event == 'exception': + return self.dispatch_exception(frame, arg) + if event == 'c_call': + return self.trace_dispatch + if event == 'c_exception': + return self.trace_dispatch + if event == 'c_return': + return self.trace_dispatch + if event == 'opcode': + return self.dispatch_opcode(frame, arg) + print('bdb.Bdb.dispatch: unknown debugging event:', repr(event)) return self.trace_dispatch - print('bdb.Bdb.dispatch: unknown debugging event:', repr(event)) - return self.trace_dispatch def dispatch_line(self, frame): """Invoke user function and return trace function for line event. @@ -110,7 +126,12 @@ def dispatch_line(self, frame): self.user_line(). Raise BdbQuit if self.quitting is set. Return self.trace_dispatch to continue tracing in this scope. """ - if self.stop_here(frame) or self.break_here(frame): + # GH-136057 + # For line events, we don't want to stop at the same line where + # the latest next/step command was issued. + if (self.stop_here(frame) or self.break_here(frame)) and not ( + self.cmdframe == frame and self.cmdlineno == frame.f_lineno + ): self.user_line(frame) if self.quitting: raise BdbQuit return self.trace_dispatch @@ -157,6 +178,11 @@ def dispatch_return(self, frame, arg): # The user issued a 'next' or 'until' command. if self.stopframe is frame and self.stoplineno != -1: self._set_stopinfo(None, None) + # The previous frame might not have f_trace set, unless we are + # issuing a command that does not expect to stop, we should set + # f_trace + if self.stoplineno != -1: + self._set_caller_tracefunc(frame) return self.trace_dispatch def dispatch_exception(self, frame, arg): @@ -186,6 +212,17 @@ def dispatch_exception(self, frame, arg): return self.trace_dispatch + def dispatch_opcode(self, frame, arg): + """Invoke user function and return trace function for opcode event. + If the debugger stops on the current opcode, invoke + self.user_opcode(). Raise BdbQuit if self.quitting is set. + Return self.trace_dispatch to continue tracing in this scope. + """ + if self.stop_here(frame) or self.break_here(frame): + self.user_opcode(frame) + if self.quitting: raise BdbQuit + return self.trace_dispatch + # Normally derived classes don't override the following # methods, but they may if they want to redefine the # definition of stopping and breakpoints. @@ -272,7 +309,22 @@ def user_exception(self, frame, exc_info): """Called when we stop on an exception.""" pass - def _set_stopinfo(self, stopframe, returnframe, stoplineno=0): + def user_opcode(self, frame): + """Called when we are about to execute an opcode.""" + pass + + def _set_trace_opcodes(self, trace_opcodes): + if trace_opcodes != self.trace_opcodes: + self.trace_opcodes = trace_opcodes + frame = self.enterframe + while frame is not None: + frame.f_trace_opcodes = trace_opcodes + if frame is self.botframe: + break + frame = frame.f_back + + def _set_stopinfo(self, stopframe, returnframe, stoplineno=0, opcode=False, + cmdframe=None, cmdlineno=None): """Set the attributes for stopping. If stoplineno is greater than or equal to 0, then stop at line @@ -285,6 +337,21 @@ def _set_stopinfo(self, stopframe, returnframe, stoplineno=0): # stoplineno >= 0 means: stop at line >= the stoplineno # stoplineno -1 means: don't stop at all self.stoplineno = stoplineno + # cmdframe/cmdlineno is the frame/line number when the user issued + # step/next commands. + self.cmdframe = cmdframe + self.cmdlineno = cmdlineno + self._set_trace_opcodes(opcode) + + def _set_caller_tracefunc(self, current_frame): + # Issue #13183: pdb skips frames after hitting a breakpoint and running + # step commands. + # Restore the trace function in the caller (that may not have been set + # for performance reasons) when returning from the current frame, unless + # the caller is the botframe. + caller_frame = current_frame.f_back + if caller_frame and not caller_frame.f_trace and caller_frame is not self.botframe: + caller_frame.f_trace = self.trace_dispatch # Derived classes and clients can call the following methods # to affect the stepping state. @@ -299,19 +366,17 @@ def set_until(self, frame, lineno=None): def set_step(self): """Stop after one line of code.""" - # Issue #13183: pdb skips frames after hitting a breakpoint and running - # step commands. - # Restore the trace function in the caller (that may not have been set - # for performance reasons) when returning from the current frame. - if self.frame_returning: - caller_frame = self.frame_returning.f_back - if caller_frame and not caller_frame.f_trace: - caller_frame.f_trace = self.trace_dispatch - self._set_stopinfo(None, None) + # set_step() could be called from signal handler so enterframe might be None + self._set_stopinfo(None, None, cmdframe=self.enterframe, + cmdlineno=getattr(self.enterframe, 'f_lineno', None)) + + def set_stepinstr(self): + """Stop before the next instruction.""" + self._set_stopinfo(None, None, opcode=True) def set_next(self, frame): """Stop on the next line in or below the given frame.""" - self._set_stopinfo(frame, None) + self._set_stopinfo(frame, None, cmdframe=frame, cmdlineno=frame.f_lineno) def set_return(self, frame): """Stop when returning from the given frame.""" @@ -328,11 +393,15 @@ def set_trace(self, frame=None): if frame is None: frame = sys._getframe().f_back self.reset() - while frame: - frame.f_trace = self.trace_dispatch - self.botframe = frame - frame = frame.f_back - self.set_step() + with self.set_enterframe(frame): + while frame: + frame.f_trace = self.trace_dispatch + self.botframe = frame + self.frame_trace_lines_opcodes[frame] = (frame.f_trace_lines, frame.f_trace_opcodes) + # We need f_trace_lines == True for the debugger to work + frame.f_trace_lines = True + frame = frame.f_back + self.set_stepinstr() sys.settrace(self.trace_dispatch) def set_continue(self): @@ -349,6 +418,9 @@ def set_continue(self): while frame and frame is not self.botframe: del frame.f_trace frame = frame.f_back + for frame, (trace_lines, trace_opcodes) in self.frame_trace_lines_opcodes.items(): + frame.f_trace_lines, frame.f_trace_opcodes = trace_lines, trace_opcodes + self.frame_trace_lines_opcodes = {} def set_quit(self): """Set quitting attribute to True. @@ -387,6 +459,14 @@ def set_break(self, filename, lineno, temporary=False, cond=None, return 'Line %s:%d does not exist' % (filename, lineno) self._add_to_breaks(filename, lineno) bp = Breakpoint(filename, lineno, temporary, cond, funcname) + # After we set a new breakpoint, we need to search through all frames + # and set f_trace to trace_dispatch if there could be a breakpoint in + # that frame. + frame = self.enterframe + while frame: + if self.break_anywhere(frame): + frame.f_trace = self.trace_dispatch + frame = frame.f_back return None def _load_breaks(self): diff --git a/Lib/test/test_bdb.py b/Lib/test/test_bdb.py index a3abbbb8db2..d1c1c786861 100644 --- a/Lib/test/test_bdb.py +++ b/Lib/test/test_bdb.py @@ -228,6 +228,10 @@ def user_exception(self, frame, exc_info): self.process_event('exception', frame) self.next_set_method() + def user_opcode(self, frame): + self.process_event('opcode', frame) + self.next_set_method() + def do_clear(self, arg): # The temporary breakpoints are deleted in user_line(). bp_list = [self.currentbp] @@ -366,7 +370,7 @@ def next_set_method(self): set_method = getattr(self, 'set_' + set_type) # The following set methods give back control to the tracer. - if set_type in ('step', 'continue', 'quit'): + if set_type in ('step', 'stepinstr', 'continue', 'quit'): set_method() return elif set_type in ('next', 'return'): @@ -586,7 +590,7 @@ def fail(self, msg=None): class StateTestCase(BaseTestCase): """Test the step, next, return, until and quit 'set_' methods.""" - @unittest.skip("TODO: RUSTPYTHON, Error in atexit._run_exitfuncs") + @unittest.skip('TODO: RUSTPYTHON; Error in atexit._run_exitfuncs') def test_step(self): self.expect_set = [ ('line', 2, 'tfunc_main'), ('step', ), @@ -597,7 +601,7 @@ def test_step(self): with TracerRun(self) as tracer: tracer.runcall(tfunc_main) - @unittest.skip("TODO: RUSTPYTHON, Error in atexit._run_exitfuncs") + @unittest.skip('TODO: RUSTPYTHON; Error in atexit._run_exitfuncs') def test_step_next_on_last_statement(self): for set_type in ('step', 'next'): with self.subTest(set_type=set_type): @@ -612,7 +616,18 @@ def test_step_next_on_last_statement(self): with TracerRun(self) as tracer: tracer.runcall(tfunc_main) - @unittest.skip("TODO: RUSTPYTHON, Error in atexit._run_exitfuncs") + @unittest.skip('TODO: RUSTPYTHON') + # AssertionError: All paired tuples have not been processed, the last one was number 1 [('next',), ('quit',)] + def test_stepinstr(self): + self.expect_set = [ + ('line', 2, 'tfunc_main'), ('stepinstr', ), + ('opcode', 2, 'tfunc_main'), ('next', ), + ('line', 3, 'tfunc_main'), ('quit', ), + ] + with TracerRun(self) as tracer: + tracer.runcall(tfunc_main) + + @unittest.skip('TODO: RUSTPYTHON; Error in atexit._run_exitfuncs') def test_next(self): self.expect_set = [ ('line', 2, 'tfunc_main'), ('step', ), @@ -624,7 +639,7 @@ def test_next(self): with TracerRun(self) as tracer: tracer.runcall(tfunc_main) - @unittest.skip("TODO: RUSTPYTHON, Error in atexit._run_exitfuncs") + @unittest.skip('TODO: RUSTPYTHON; Error in atexit._run_exitfuncs') def test_next_over_import(self): code = """ def main(): @@ -639,7 +654,7 @@ def main(): with TracerRun(self) as tracer: tracer.runcall(tfunc_import) - @unittest.skip("TODO: RUSTPYTHON, Error in atexit._run_exitfuncs") + @unittest.skip('TODO: RUSTPYTHON; Error in atexit._run_exitfuncs') def test_next_on_plain_statement(self): # Check that set_next() is equivalent to set_step() on a plain # statement. @@ -652,7 +667,7 @@ def test_next_on_plain_statement(self): with TracerRun(self) as tracer: tracer.runcall(tfunc_main) - @unittest.skip("TODO: RUSTPYTHON, Error in atexit._run_exitfuncs") + @unittest.skip('TODO: RUSTPYTHON; Error in atexit._run_exitfuncs') def test_next_in_caller_frame(self): # Check that set_next() in the caller frame causes the tracer # to stop next in the caller frame. @@ -666,7 +681,7 @@ def test_next_in_caller_frame(self): with TracerRun(self) as tracer: tracer.runcall(tfunc_main) - @unittest.skip("TODO: RUSTPYTHON, Error in atexit._run_exitfuncs") + @unittest.skip('TODO: RUSTPYTHON; Error in atexit._run_exitfuncs') def test_return(self): self.expect_set = [ ('line', 2, 'tfunc_main'), ('step', ), @@ -679,7 +694,7 @@ def test_return(self): with TracerRun(self) as tracer: tracer.runcall(tfunc_main) - @unittest.skip("TODO: RUSTPYTHON, Error in atexit._run_exitfuncs") + @unittest.skip('TODO: RUSTPYTHON; Error in atexit._run_exitfuncs') def test_return_in_caller_frame(self): self.expect_set = [ ('line', 2, 'tfunc_main'), ('step', ), @@ -691,7 +706,7 @@ def test_return_in_caller_frame(self): with TracerRun(self) as tracer: tracer.runcall(tfunc_main) - @unittest.skip("TODO: RUSTPYTHON, Error in atexit._run_exitfuncs") + @unittest.skip('TODO: RUSTPYTHON; Error in atexit._run_exitfuncs') def test_until(self): self.expect_set = [ ('line', 2, 'tfunc_main'), ('step', ), @@ -703,7 +718,7 @@ def test_until(self): with TracerRun(self) as tracer: tracer.runcall(tfunc_main) - @unittest.skip("TODO: RUSTPYTHON, Error in atexit._run_exitfuncs") + @unittest.skip('TODO: RUSTPYTHON; Error in atexit._run_exitfuncs') def test_until_with_too_large_count(self): self.expect_set = [ ('line', 2, 'tfunc_main'), break_in_func('tfunc_first'), @@ -714,7 +729,7 @@ def test_until_with_too_large_count(self): with TracerRun(self) as tracer: tracer.runcall(tfunc_main) - @unittest.skip("TODO: RUSTPYTHON, Error in atexit._run_exitfuncs") + @unittest.skip('TODO: RUSTPYTHON; Error in atexit._run_exitfuncs') def test_until_in_caller_frame(self): self.expect_set = [ ('line', 2, 'tfunc_main'), ('step', ), @@ -726,7 +741,8 @@ def test_until_in_caller_frame(self): with TracerRun(self) as tracer: tracer.runcall(tfunc_main) - @unittest.skip("TODO: RUSTPYTHON, Error in atexit._run_exitfuncs") + @unittest.skip('TODO: RUSTPYTHON; Error in atexit._run_exitfuncs') + @patch_list(sys.meta_path) def test_skip(self): # Check that tracing is skipped over the import statement in # 'tfunc_import()'. @@ -759,7 +775,7 @@ def test_skip_with_no_name_module(self): bdb = Bdb(skip=['anything*']) self.assertIs(bdb.is_skipped_module(None), False) - @unittest.skip("TODO: RUSTPYTHON, Error in atexit._run_exitfuncs") + @unittest.skip('TODO: RUSTPYTHON; Error in atexit._run_exitfuncs') def test_down(self): # Check that set_down() raises BdbError at the newest frame. self.expect_set = [ @@ -768,7 +784,7 @@ def test_down(self): with TracerRun(self) as tracer: self.assertRaises(BdbError, tracer.runcall, tfunc_main) - @unittest.skip("TODO: RUSTPYTHON, Error in atexit._run_exitfuncs") + @unittest.skip('TODO: RUSTPYTHON; Error in atexit._run_exitfuncs') def test_up(self): self.expect_set = [ ('line', 2, 'tfunc_main'), ('step', ), @@ -782,7 +798,7 @@ def test_up(self): class BreakpointTestCase(BaseTestCase): """Test the breakpoint set method.""" - @unittest.skip("TODO: RUSTPYTHON, Error in atexit._run_exitfuncs") + @unittest.skip('TODO: RUSTPYTHON; Error in atexit._run_exitfuncs') def test_bp_on_non_existent_module(self): self.expect_set = [ ('line', 2, 'tfunc_import'), ('break', ('/non/existent/module.py', 1)) @@ -790,7 +806,7 @@ def test_bp_on_non_existent_module(self): with TracerRun(self) as tracer: self.assertRaises(BdbError, tracer.runcall, tfunc_import) - @unittest.skip("TODO: RUSTPYTHON, Error in atexit._run_exitfuncs") + @unittest.skip('TODO: RUSTPYTHON; Error in atexit._run_exitfuncs') def test_bp_after_last_statement(self): code = """ def main(): @@ -804,7 +820,7 @@ def main(): with TracerRun(self) as tracer: self.assertRaises(BdbError, tracer.runcall, tfunc_import) - @unittest.skip("TODO: RUSTPYTHON, Error in atexit._run_exitfuncs") + @unittest.skip('TODO: RUSTPYTHON; Error in atexit._run_exitfuncs') def test_temporary_bp(self): code = """ def func(): @@ -828,7 +844,7 @@ def main(): with TracerRun(self) as tracer: tracer.runcall(tfunc_import) - @unittest.skip("TODO: RUSTPYTHON, Error in atexit._run_exitfuncs") + @unittest.skip('TODO: RUSTPYTHON; Error in atexit._run_exitfuncs') def test_disabled_temporary_bp(self): code = """ def func(): @@ -857,7 +873,7 @@ def main(): with TracerRun(self) as tracer: tracer.runcall(tfunc_import) - @unittest.skip("TODO: RUSTPYTHON, Error in atexit._run_exitfuncs") + @unittest.skip('TODO: RUSTPYTHON; Error in atexit._run_exitfuncs') def test_bp_condition(self): code = """ def func(a): @@ -878,7 +894,7 @@ def main(): with TracerRun(self) as tracer: tracer.runcall(tfunc_import) - @unittest.skip("TODO: RUSTPYTHON, Error in atexit._run_exitfuncs") + @unittest.skip('TODO: RUSTPYTHON; Error in atexit._run_exitfuncs') def test_bp_exception_on_condition_evaluation(self): code = """ def func(a): @@ -898,7 +914,7 @@ def main(): with TracerRun(self) as tracer: tracer.runcall(tfunc_import) - @unittest.skip("TODO: RUSTPYTHON, Error in atexit._run_exitfuncs") + @unittest.skip('TODO: RUSTPYTHON; Error in atexit._run_exitfuncs') def test_bp_ignore_count(self): code = """ def func(): @@ -920,7 +936,7 @@ def main(): with TracerRun(self) as tracer: tracer.runcall(tfunc_import) - @unittest.skip("TODO: RUSTPYTHON, Error in atexit._run_exitfuncs") + @unittest.skip('TODO: RUSTPYTHON; Error in atexit._run_exitfuncs') def test_ignore_count_on_disabled_bp(self): code = """ def func(): @@ -948,7 +964,7 @@ def main(): with TracerRun(self) as tracer: tracer.runcall(tfunc_import) - @unittest.skip("TODO: RUSTPYTHON, Error in atexit._run_exitfuncs") + @unittest.skip('TODO: RUSTPYTHON; Error in atexit._run_exitfuncs') def test_clear_two_bp_on_same_line(self): code = """ def func(): @@ -974,7 +990,7 @@ def main(): with TracerRun(self) as tracer: tracer.runcall(tfunc_import) - @unittest.skip("TODO: RUSTPYTHON, Error in atexit._run_exitfuncs") + @unittest.skip('TODO: RUSTPYTHON; Error in atexit._run_exitfuncs') def test_clear_at_no_bp(self): self.expect_set = [ ('line', 2, 'tfunc_import'), ('clear', (__file__, 1)) @@ -1028,7 +1044,7 @@ def test_load_bps_from_previous_Bdb_instance(self): class RunTestCase(BaseTestCase): """Test run, runeval and set_trace.""" - @unittest.skip("TODO: RUSTPYTHON, Error in atexit._run_exitfuncs") + @unittest.skip('TODO: RUSTPYTHON; Error in atexit._run_exitfuncs') def test_run_step(self): # Check that the bdb 'run' method stops at the first line event. code = """ @@ -1041,7 +1057,7 @@ def test_run_step(self): with TracerRun(self) as tracer: tracer.run(compile(textwrap.dedent(code), '', 'exec')) - @unittest.skip("TODO: RUSTPYTHON, Error in atexit._run_exitfuncs") + @unittest.skip('TODO: RUSTPYTHON; Error in atexit._run_exitfuncs') def test_runeval_step(self): # Test bdb 'runeval'. code = """ @@ -1064,7 +1080,7 @@ def main(): class IssuesTestCase(BaseTestCase): """Test fixed bdb issues.""" - @unittest.skip("TODO: RUSTPYTHON, Error in atexit._run_exitfuncs") + @unittest.skip('TODO: RUSTPYTHON; Error in atexit._run_exitfuncs') def test_step_at_return_with_no_trace_in_caller(self): # Issue #13183. # Check that the tracer does step into the caller frame when the @@ -1095,7 +1111,7 @@ def func(): with TracerRun(self) as tracer: tracer.runcall(tfunc_import) - @unittest.skip("TODO: RUSTPYTHON, Error in atexit._run_exitfuncs") + @unittest.skip('TODO: RUSTPYTHON; Error in atexit._run_exitfuncs') def test_next_until_return_in_generator(self): # Issue #16596. # Check that set_next(), set_until() and set_return() do not treat the @@ -1137,7 +1153,7 @@ def main(): with TracerRun(self) as tracer: tracer.runcall(tfunc_import) - @unittest.skip("TODO: RUSTPYTHON, Error in atexit._run_exitfuncs") + @unittest.skip('TODO: RUSTPYTHON; Error in atexit._run_exitfuncs') def test_next_command_in_generator_for_loop(self): # Issue #16596. code = """ @@ -1169,7 +1185,7 @@ def main(): with TracerRun(self) as tracer: tracer.runcall(tfunc_import) - @unittest.skip("TODO: RUSTPYTHON, Error in atexit._run_exitfuncs") + @unittest.skip('TODO: RUSTPYTHON; Error in atexit._run_exitfuncs') def test_next_command_in_generator_with_subiterator(self): # Issue #16596. code = """ @@ -1201,7 +1217,7 @@ def main(): with TracerRun(self) as tracer: tracer.runcall(tfunc_import) - @unittest.skip("TODO: RUSTPYTHON, Error in atexit._run_exitfuncs") + @unittest.skip('TODO: RUSTPYTHON; Error in atexit._run_exitfuncs') def test_return_command_in_generator_with_subiterator(self): # Issue #16596. code = """ @@ -1233,6 +1249,21 @@ def main(): with TracerRun(self) as tracer: tracer.runcall(tfunc_import) + @unittest.skip('TODO: RUSTPYTHON') + # AssertionError: All paired tuples have not been processed, the last one was number 1 [('next',)] + def test_next_to_botframe(self): + # gh-125422 + # Check that next command won't go to the bottom frame. + code = """ + lno = 2 + """ + self.expect_set = [ + ('line', 2, ''), ('step', ), + ('return', 2, ''), ('next', ), + ] + with TracerRun(self) as tracer: + tracer.run(compile(textwrap.dedent(code), '', 'exec')) + class TestRegressions(unittest.TestCase): def test_format_stack_entry_no_lineno(self): From e79a1a1a6665565530bc143a305496b5cefed92e Mon Sep 17 00:00:00 2001 From: "Jeong, YunWon" <69878+youknowone@users.noreply.github.com> Date: Tue, 30 Dec 2025 16:58:53 +0900 Subject: [PATCH 0006/1575] Fix traceback (#6569) * Fix traceback * Update traceback from CPython 3.13.11 * unmark test_traceback * fix code * fix debug range * fix tests --- .cspell.dict/python-more.txt | 1 + Lib/test/test_code.py | 4 - Lib/test/test_traceback.py | 110 --- Lib/test/test_zipimport.py | 2 + Lib/traceback.py | 1198 +++++++++++++++++++++++--- crates/codegen/src/compile.rs | 30 +- crates/codegen/src/ir.rs | 87 +- crates/compiler-core/src/bytecode.rs | 6 +- crates/compiler-core/src/marshal.rs | 19 +- crates/vm/src/builtins/code.rs | 21 +- crates/vm/src/builtins/frame.rs | 11 +- crates/vm/src/frame.rs | 6 +- crates/vm/src/vm/mod.rs | 1 + crates/vm/src/vm/setting.rs | 4 +- examples/dis.rs | 5 +- src/settings.rs | 4 + 16 files changed, 1202 insertions(+), 307 deletions(-) diff --git a/.cspell.dict/python-more.txt b/.cspell.dict/python-more.txt index d381bfe1e03..1f3fc4864cd 100644 --- a/.cspell.dict/python-more.txt +++ b/.cspell.dict/python-more.txt @@ -178,6 +178,7 @@ PYTHONHASHSEED PYTHONHOME PYTHONINSPECT PYTHONINTMAXSTRDIGITS +PYTHONNODEBUGRANGES PYTHONNOUSERSITE PYTHONOPTIMIZE PYTHONPATH diff --git a/Lib/test/test_code.py b/Lib/test/test_code.py index 804cce1dba4..f2ef233a59a 100644 --- a/Lib/test/test_code.py +++ b/Lib/test/test_code.py @@ -425,8 +425,6 @@ def test_co_positions_artificial_instructions(self): ] ) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_endline_and_columntable_none_when_no_debug_ranges(self): # Make sure that if `-X no_debug_ranges` is used, there is # minimal debug info @@ -442,8 +440,6 @@ def f(): """) assert_python_ok('-X', 'no_debug_ranges', '-c', code) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_endline_and_columntable_none_when_no_debug_ranges_env(self): # Same as above but using the environment variable opt out. code = textwrap.dedent(""" diff --git a/Lib/test/test_traceback.py b/Lib/test/test_traceback.py index 9d95903d526..f2ec8344b2f 100644 --- a/Lib/test/test_traceback.py +++ b/Lib/test/test_traceback.py @@ -218,8 +218,6 @@ def test_base_exception(self): lst = traceback.format_exception_only(e.__class__, e) self.assertEqual(lst, ['KeyboardInterrupt\n']) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_format_exception_only_bad__str__(self): class X(Exception): def __str__(self): @@ -238,8 +236,6 @@ def test_format_exception_group_without_show_group(self): err = traceback.format_exception_only(eg) self.assertEqual(err, ['ExceptionGroup: A (1 sub-exception)\n']) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_format_exception_group(self): eg = ExceptionGroup('A', [ValueError('B')]) err = traceback.format_exception_only(eg, show_group=True) @@ -248,8 +244,6 @@ def test_format_exception_group(self): ' ValueError: B\n', ]) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_format_base_exception_group(self): eg = BaseExceptionGroup('A', [BaseException('B')]) err = traceback.format_exception_only(eg, show_group=True) @@ -258,8 +252,6 @@ def test_format_base_exception_group(self): ' BaseException: B\n', ]) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_format_exception_group_with_note(self): exc = ValueError('B') exc.add_note('Note') @@ -271,8 +263,6 @@ def test_format_exception_group_with_note(self): ' Note\n', ]) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_format_exception_group_explicit_class(self): eg = ExceptionGroup('A', [ValueError('B')]) err = traceback.format_exception_only(ExceptionGroup, eg, show_group=True) @@ -281,8 +271,6 @@ def test_format_exception_group_explicit_class(self): ' ValueError: B\n', ]) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_format_exception_group_multiple_exceptions(self): eg = ExceptionGroup('A', [ValueError('B'), TypeError('C')]) err = traceback.format_exception_only(eg, show_group=True) @@ -292,8 +280,6 @@ def test_format_exception_group_multiple_exceptions(self): ' TypeError: C\n', ]) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_format_exception_group_multiline_messages(self): eg = ExceptionGroup('A\n1', [ValueError('B\n2')]) err = traceback.format_exception_only(eg, show_group=True) @@ -303,8 +289,6 @@ def test_format_exception_group_multiline_messages(self): ' 2\n', ]) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_format_exception_group_multiline2_messages(self): exc = ValueError('B\n\n2\n') exc.add_note('\nC\n\n3') @@ -323,8 +307,6 @@ def test_format_exception_group_multiline2_messages(self): ' IndexError: D\n', ]) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_format_exception_group_syntax_error(self): exc = SyntaxError("error", ("x.py", 23, None, "bad syntax")) eg = ExceptionGroup('A\n1', [exc]) @@ -336,8 +318,6 @@ def test_format_exception_group_syntax_error(self): ' SyntaxError: error\n', ]) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_format_exception_group_nested_with_notes(self): exc = IndexError('D') exc.add_note('Note\nmultiline') @@ -358,8 +338,6 @@ def test_format_exception_group_nested_with_notes(self): ' TypeError: F\n', ]) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_format_exception_group_with_tracebacks(self): def f(): try: @@ -385,8 +363,6 @@ def g(): ' TypeError: g\n', ]) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_format_exception_group_with_cause(self): def f(): try: @@ -404,8 +380,6 @@ def f(): ' ValueError: 0\n', ]) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_format_exception_group_syntax_error_with_custom_values(self): # See https://github.com/python/cpython/issues/128894 for exc in [ @@ -550,16 +524,12 @@ def test_print_exception_exc(self): traceback.print_exception(Exception("projector"), file=output) self.assertEqual(output.getvalue(), "Exception: projector\n") - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_print_last(self): with support.swap_attr(sys, 'last_exc', ValueError(42)): output = StringIO() traceback.print_last(file=output) self.assertEqual(output.getvalue(), "ValueError: 42\n") - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_format_exception_exc(self): e = Exception("projector") output = traceback.format_exception(e) @@ -598,8 +568,6 @@ def test_exception_is_None(self): self.assertEqual( traceback.format_exception_only(None, None), [NONE_EXC_STRING]) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_signatures(self): self.assertEqual( str(inspect.signature(traceback.print_exception)), @@ -2286,8 +2254,6 @@ def test_print_exception_bad_type_capi(self): 'Exception expected for value, int found\n') ) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_print_exception_bad_type_python(self): msg = "Exception expected for value, int found" with self.assertRaisesRegex(TypeError, msg): @@ -2366,8 +2332,6 @@ def test_simple(self): self.assertTrue(lines[1].startswith(' File')) self.assertIn('1/0 # Marker', lines[2]) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_cause(self): def inner_raise(): try: @@ -2501,8 +2465,6 @@ def test_message_none(self): err = self.get_report(Exception('')) self.assertIn('Exception\n', err) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_syntax_error_various_offsets(self): for offset in range(-5, 10): for add in [0, 2]: @@ -2525,8 +2487,6 @@ def test_syntax_error_various_offsets(self): exp = "\n".join(expected) self.assertEqual(exp, err) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_exception_with_note(self): e = ValueError(123) vanilla = self.get_report(e) @@ -2545,8 +2505,6 @@ def test_exception_with_note(self): del e.__notes__ self.assertEqual(self.get_report(e), vanilla) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_exception_with_invalid_notes(self): e = ValueError(123) vanilla = self.get_report(e) @@ -2604,8 +2562,6 @@ def __getattr__(self, name): self.get_report(e), vanilla + "Ignored error getting __notes__: ValueError('no __notes__')\n") - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_exception_with_multiple_notes(self): for e in [ValueError(42), SyntaxError('bad syntax')]: with self.subTest(e=e): @@ -2685,8 +2641,6 @@ def __str__(self): exp = f'.{X.__qualname__}: I am X\n' self.assertEqual(exp, err) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_exception_bad__str__(self): class X(Exception): def __str__(self): @@ -2861,8 +2815,6 @@ def exc(): report = self.get_report(exc) self.assertEqual(report, expected) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_exception_group_width_limit(self): excs = [] for i in range(1000): @@ -2907,8 +2859,6 @@ def test_exception_group_width_limit(self): report = self.get_report(eg) self.assertEqual(report, expected) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_exception_group_depth_limit(self): exc = TypeError('bad type') for i in range(1000): @@ -3371,8 +3321,6 @@ def test_basics(self): self.assertNotEqual(f, object()) self.assertEqual(f, ALWAYS_EQ) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_lazy_lines(self): linecache.clearcache() f = traceback.FrameSummary("f", 1, "dummy", lookup_line=False) @@ -3509,8 +3457,6 @@ def some_inner(): s.format(), [f'{__file__}:{some_inner.__code__.co_firstlineno + 1}']) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_dropping_frames(self): def f(): 1/0 @@ -3611,13 +3557,9 @@ def do_test_smoke(self, exc, expected_type_str): self.assertEqual(expected_type_str, exc.exc_type_str) self.assertEqual(str(exc_obj), str(exc)) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_smoke_builtin(self): self.do_test_smoke(ValueError(42), 'ValueError') - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_smoke_user_exception(self): class MyException(Exception): pass @@ -3630,8 +3572,6 @@ class MyException(Exception): 'test_smoke_user_exception..MyException') self.do_test_smoke(MyException('bad things happened'), expected) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_from_exception(self): # Check all the parameters are accepted. def foo(): @@ -3657,8 +3597,6 @@ def foo(): self.assertEqual(type(exc_obj).__name__, exc.exc_type_str) self.assertEqual(str(exc_obj), str(exc)) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_cause(self): try: try: @@ -3683,8 +3621,6 @@ def test_cause(self): self.assertEqual(type(exc_obj).__name__, exc.exc_type_str) self.assertEqual(str(exc_obj), str(exc)) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_context(self): try: try: @@ -3734,8 +3670,6 @@ def f(): self.assertIn( "RecursionError: maximum recursion depth exceeded", res[-1]) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_compact_with_cause(self): try: try: @@ -3758,8 +3692,6 @@ def test_compact_with_cause(self): self.assertEqual(type(exc_obj).__name__, exc.exc_type_str) self.assertEqual(str(exc_obj), str(exc)) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_compact_no_cause(self): try: try: @@ -3782,8 +3714,6 @@ def test_compact_no_cause(self): self.assertEqual(type(exc_obj).__name__, exc.exc_type_str) self.assertEqual(str(exc_obj), str(exc)) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_no_save_exc_type(self): try: 1/0 @@ -3912,8 +3842,6 @@ def test_lookup_lines(self): linecache.updatecache('/foo.py', globals()) self.assertEqual(exc.stack[0].line, "import sys") - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_locals(self): linecache.updatecache('/foo.py', globals()) e = Exception("uh oh") @@ -3962,8 +3890,6 @@ def f(): 'ZeroDivisionError: division by zero', '']) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_dont_swallow_cause_or_context_of_falsey_exception(self): # see gh-132308: Ensure that __cause__ or __context__ attributes of exceptions # that evaluate as falsey are included in the output. For falsey term, @@ -4034,8 +3960,6 @@ def test_exception_group_format_exception_only(self): self.assertEqual(formatted, expected) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_exception_group_format_exception_onlyi_recursive(self): teg = traceback.TracebackException.from_exception(self.eg) formatted = ''.join(teg.format_exception_only(show_group=True)).split('\n') @@ -4100,8 +4024,6 @@ def test_exception_group_format(self): self.assertEqual(formatted, expected) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_max_group_width(self): excs1 = [] excs2 = [] @@ -4140,8 +4062,6 @@ def test_max_group_width(self): self.assertEqual(formatted, expected) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_max_group_depth(self): exc = TypeError('bad type') for i in range(3): @@ -4191,8 +4111,6 @@ def test_comparison(self): self.assertNotEqual(exc, object()) self.assertEqual(exc, ALWAYS_EQ) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_dont_swallow_subexceptions_of_falsey_exceptiongroup(self): # see gh-132308: Ensure that subexceptions of exception groups # that evaluate as falsey are displayed in the output. For falsey term, @@ -4230,8 +4148,6 @@ def callable(): ) return result_lines[0] - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_getattr_suggestions(self): class Substitution: noise = more_noise = a = bc = None @@ -4275,8 +4191,6 @@ class CaseChangeOverSubstitution: actual = self.get_suggestion(cls(), 'bluch') self.assertIn(suggestion, actual) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_getattr_suggestions_underscored(self): class A: bluch = None @@ -4330,8 +4244,6 @@ class A: actual = self.get_suggestion(A(), 'bluch') self.assertNotIn("blech", actual) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_getattr_suggestions_no_args(self): class A: blech = None @@ -4349,8 +4261,6 @@ def __getattr__(self, attr): actual = self.get_suggestion(A(), 'bluch') self.assertIn("blech", actual) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_getattr_suggestions_invalid_args(self): class NonStringifyClass: __str__ = None @@ -4392,8 +4302,6 @@ def __dir__(self): self.assertNotIn("blech", actual) self.assertNotIn("oh no!", actual) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_attribute_error_with_non_string_candidates(self): class T: bluch = 1 @@ -4567,8 +4475,6 @@ def raise_attribute_error_with_bad_name(): ) self.assertNotIn("?", result_lines[-1]) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_name_error_suggestions(self): def Substitution(): noise = more_noise = a = bc = None @@ -4609,24 +4515,18 @@ def EliminationOverAddition(): actual = self.get_suggestion(func) self.assertIn(suggestion, actual) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_name_error_suggestions_from_globals(self): def func(): print(global_for_suggestio) actual = self.get_suggestion(func) self.assertIn("'global_for_suggestions'?", actual) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_name_error_suggestions_from_builtins(self): def func(): print(ZeroDivisionErrrrr) actual = self.get_suggestion(func) self.assertIn("'ZeroDivisionError'?", actual) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_name_error_suggestions_from_builtins_when_builtins_is_module(self): def func(): custom_globals = globals().copy() @@ -4635,8 +4535,6 @@ def func(): actual = self.get_suggestion(func) self.assertIn("'ZeroDivisionError'?", actual) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_name_error_suggestions_with_non_string_candidates(self): def func(): abc = 1 @@ -4785,8 +4683,6 @@ def func(): actual = self.get_suggestion(func) self.assertNotIn("blech", actual) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_name_error_with_instance(self): class A: def __init__(self): @@ -4843,8 +4739,6 @@ def func(): actual = self.get_suggestion(func) self.assertNotIn("something", actual) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_name_error_for_stdlib_modules(self): def func(): stream = io.StringIO() @@ -4852,8 +4746,6 @@ def func(): actual = self.get_suggestion(func) self.assertIn("forget to import 'io'", actual) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_name_error_for_private_stdlib_modules(self): def func(): stream = _io.StringIO() @@ -4898,8 +4790,6 @@ def test_all(self): expected.add(name) self.assertCountEqual(traceback.__all__, expected) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_levenshtein_distance(self): # copied from _testinternalcapi.test_edit_cost # to also exercise the Python implementation diff --git a/Lib/test/test_zipimport.py b/Lib/test/test_zipimport.py index b291d530169..b9c1fd6134e 100644 --- a/Lib/test/test_zipimport.py +++ b/Lib/test/test_zipimport.py @@ -724,6 +724,8 @@ def doTraceback(self, module): else: raise AssertionError("This ought to be impossible") + # TODO: RUSTPYTHON; empty caret lines from equal col/end_col + @unittest.expectedFailure def testTraceback(self): files = {TESTMOD + ".py": (NOW, raise_src)} self.doTest(None, files, TESTMOD, call=self.doTraceback) diff --git a/Lib/traceback.py b/Lib/traceback.py index d6a010f4157..572a3177cb0 100644 --- a/Lib/traceback.py +++ b/Lib/traceback.py @@ -1,9 +1,14 @@ """Extract, format and print information about Python stack traces.""" -import collections +import collections.abc import itertools import linecache import sys +import textwrap +import warnings +from contextlib import suppress +import _colorize +from _colorize import ANSIColors __all__ = ['extract_stack', 'extract_tb', 'format_exception', 'format_exception_only', 'format_list', 'format_stack', @@ -16,6 +21,7 @@ # Formatting and printing lists of traceback lines. # + def print_list(extracted_list, file=None): """Print the list of tuples as returned by extract_tb() or extract_stack() as a formatted stack trace to the given file.""" @@ -69,7 +75,8 @@ def extract_tb(tb, limit=None): trace. The line is a string with leading and trailing whitespace stripped; if the source is not available it is None. """ - return StackSummary.extract(walk_tb(tb), limit=limit) + return StackSummary._extract_from_extended_frame_gen( + _walk_tb_with_full_positions(tb), limit=limit) # # Exception formatting and output. @@ -95,14 +102,18 @@ def _parse_value_tb(exc, value, tb): raise ValueError("Both or neither of value and tb must be given") if value is tb is _sentinel: if exc is not None: - return exc, exc.__traceback__ + if isinstance(exc, BaseException): + return exc, exc.__traceback__ + + raise TypeError(f'Exception expected for value, ' + f'{type(exc).__name__} found') else: return None, None return value, tb def print_exception(exc, /, value=_sentinel, tb=_sentinel, limit=None, \ - file=None, chain=True): + file=None, chain=True, **kwargs): """Print exception up to 'limit' stack trace entries from 'tb' to 'file'. This differs from print_tb() in the following ways: (1) if @@ -113,16 +124,23 @@ def print_exception(exc, /, value=_sentinel, tb=_sentinel, limit=None, \ occurred with a caret on the next line indicating the approximate position of the error. """ + colorize = kwargs.get("colorize", False) value, tb = _parse_value_tb(exc, value, tb) - if file is None: - file = sys.stderr te = TracebackException(type(value), value, tb, limit=limit, compact=True) - for line in te.format(chain=chain): - print(line, file=file, end="") + te.print(file=file, chain=chain, colorize=colorize) + + +BUILTIN_EXCEPTION_LIMIT = object() + + +def _print_exception_bltin(exc, /): + file = sys.stderr if sys.stderr is not None else sys.__stderr__ + colorize = _colorize.can_colorize(file=file) + return print_exception(exc, limit=BUILTIN_EXCEPTION_LIMIT, file=file, colorize=colorize) def format_exception(exc, /, value=_sentinel, tb=_sentinel, limit=None, \ - chain=True): + chain=True, **kwargs): """Format a stack trace and the exception information. The arguments have the same meaning as the corresponding arguments @@ -131,64 +149,79 @@ def format_exception(exc, /, value=_sentinel, tb=_sentinel, limit=None, \ these lines are concatenated and printed, exactly the same text is printed as does print_exception(). """ + colorize = kwargs.get("colorize", False) value, tb = _parse_value_tb(exc, value, tb) te = TracebackException(type(value), value, tb, limit=limit, compact=True) - return list(te.format(chain=chain)) + return list(te.format(chain=chain, colorize=colorize)) -def format_exception_only(exc, /, value=_sentinel): +def format_exception_only(exc, /, value=_sentinel, *, show_group=False, **kwargs): """Format the exception part of a traceback. The return value is a list of strings, each ending in a newline. - Normally, the list contains a single string; however, for - SyntaxError exceptions, it contains several lines that (when - printed) display detailed information about where the syntax - error occurred. - - The message indicating which exception occurred is always the last - string in the list. + The list contains the exception's message, which is + normally a single string; however, for :exc:`SyntaxError` exceptions, it + contains several lines that (when printed) display detailed information + about where the syntax error occurred. Following the message, the list + contains the exception's ``__notes__``. + When *show_group* is ``True``, and the exception is an instance of + :exc:`BaseExceptionGroup`, the nested exceptions are included as + well, recursively, with indentation relative to their nesting depth. """ + colorize = kwargs.get("colorize", False) if value is _sentinel: value = exc te = TracebackException(type(value), value, None, compact=True) - return list(te.format_exception_only()) + return list(te.format_exception_only(show_group=show_group, colorize=colorize)) # -- not official API but folk probably use these two functions. -def _format_final_exc_line(etype, value): - valuestr = _some_str(value) - if value is None or not valuestr: - line = "%s\n" % etype +def _format_final_exc_line(etype, value, *, insert_final_newline=True, colorize=False): + valuestr = _safe_string(value, 'exception') + end_char = "\n" if insert_final_newline else "" + if colorize: + if value is None or not valuestr: + line = f"{ANSIColors.BOLD_MAGENTA}{etype}{ANSIColors.RESET}{end_char}" + else: + line = f"{ANSIColors.BOLD_MAGENTA}{etype}{ANSIColors.RESET}: {ANSIColors.MAGENTA}{valuestr}{ANSIColors.RESET}{end_char}" else: - line = "%s: %s\n" % (etype, valuestr) + if value is None or not valuestr: + line = f"{etype}{end_char}" + else: + line = f"{etype}: {valuestr}{end_char}" return line -def _some_str(value): + +def _safe_string(value, what, func=str): try: - return str(value) + return func(value) except: - return '' % type(value).__name__ + return f'<{what} {func.__name__}() failed>' # -- def print_exc(limit=None, file=None, chain=True): - """Shorthand for 'print_exception(*sys.exc_info(), limit, file)'.""" - print_exception(*sys.exc_info(), limit=limit, file=file, chain=chain) + """Shorthand for 'print_exception(sys.exception(), limit=limit, file=file, chain=chain)'.""" + print_exception(sys.exception(), limit=limit, file=file, chain=chain) def format_exc(limit=None, chain=True): """Like print_exc() but return a string.""" - return "".join(format_exception(*sys.exc_info(), limit=limit, chain=chain)) + return "".join(format_exception(sys.exception(), limit=limit, chain=chain)) def print_last(limit=None, file=None, chain=True): - """This is a shorthand for 'print_exception(sys.last_type, - sys.last_value, sys.last_traceback, limit, file)'.""" - if not hasattr(sys, "last_type"): + """This is a shorthand for 'print_exception(sys.last_exc, limit=limit, file=file, chain=chain)'.""" + if not hasattr(sys, "last_exc") and not hasattr(sys, "last_type"): raise ValueError("no last exception") - print_exception(sys.last_type, sys.last_value, sys.last_traceback, - limit, file, chain) + + if hasattr(sys, "last_exc"): + print_exception(sys.last_exc, limit=limit, file=file, chain=chain) + else: + print_exception(sys.last_type, sys.last_value, sys.last_traceback, + limit=limit, file=file, chain=chain) + # # Printing and Extracting Stacks. @@ -241,7 +274,7 @@ def clear_frames(tb): class FrameSummary: - """A single frame from a traceback. + """Information about a single frame from a traceback. - :attr:`filename` The filename for the frame. - :attr:`lineno` The line within filename for the frame that was @@ -254,10 +287,12 @@ class FrameSummary: mapping the name to the repr() of the variable. """ - __slots__ = ('filename', 'lineno', 'name', '_line', 'locals') + __slots__ = ('filename', 'lineno', 'end_lineno', 'colno', 'end_colno', + 'name', '_lines', '_lines_dedented', 'locals', '_code') def __init__(self, filename, lineno, name, *, lookup_line=True, - locals=None, line=None): + locals=None, line=None, + end_lineno=None, colno=None, end_colno=None, **kwargs): """Construct a FrameSummary. :param lookup_line: If True, `linecache` is consulted for the source @@ -269,11 +304,17 @@ def __init__(self, filename, lineno, name, *, lookup_line=True, """ self.filename = filename self.lineno = lineno + self.end_lineno = lineno if end_lineno is None else end_lineno + self.colno = colno + self.end_colno = end_colno self.name = name - self._line = line + self._code = kwargs.get("_code") + self._lines = line + self._lines_dedented = None if lookup_line: self.line - self.locals = {k: repr(v) for k, v in locals.items()} if locals else None + self.locals = {k: _safe_string(v, 'local', func=repr) + for k, v in locals.items()} if locals else None def __eq__(self, other): if isinstance(other, FrameSummary): @@ -298,13 +339,43 @@ def __repr__(self): def __len__(self): return 4 + def _set_lines(self): + if ( + self._lines is None + and self.lineno is not None + and self.end_lineno is not None + ): + lines = [] + for lineno in range(self.lineno, self.end_lineno + 1): + # treat errors (empty string) and empty lines (newline) as the same + line = linecache.getline(self.filename, lineno).rstrip() + if not line and self._code is not None and self.filename.startswith("<"): + line = linecache._getline_from_code(self._code, lineno).rstrip() + lines.append(line) + self._lines = "\n".join(lines) + "\n" + + @property + def _original_lines(self): + # Returns the line as-is from the source, without modifying whitespace. + self._set_lines() + return self._lines + + @property + def _dedented_lines(self): + # Returns _original_lines, but dedented + self._set_lines() + if self._lines_dedented is None and self._lines is not None: + self._lines_dedented = textwrap.dedent(self._lines) + return self._lines_dedented + @property def line(self): - if self._line is None: - if self.lineno is None: - return None - self._line = linecache.getline(self.filename, self.lineno) - return self._line.strip() + self._set_lines() + if self._lines is None: + return None + # return only the first line, stripped + return self._lines.partition("\n")[0].strip() + def walk_stack(f): """Walk a stack yielding the frame and line number for each frame. @@ -313,7 +384,7 @@ def walk_stack(f): current stack is used. Usually used with StackSummary.extract. """ if f is None: - f = sys._getframe().f_back.f_back + f = sys._getframe().f_back.f_back.f_back.f_back while f is not None: yield f, f.f_lineno f = f.f_back @@ -330,18 +401,40 @@ def walk_tb(tb): tb = tb.tb_next +def _walk_tb_with_full_positions(tb): + # Internal version of walk_tb that yields full code positions including + # end line and column information. + while tb is not None: + positions = _get_code_position(tb.tb_frame.f_code, tb.tb_lasti) + # Yield tb_lineno when co_positions does not have a line number to + # maintain behavior with walk_tb. + if positions[0] is None: + yield tb.tb_frame, (tb.tb_lineno, ) + positions[1:] + else: + yield tb.tb_frame, positions + tb = tb.tb_next + + +def _get_code_position(code, instruction_index): + if instruction_index < 0: + return (None, None, None, None) + positions_gen = code.co_positions() + return next(itertools.islice(positions_gen, instruction_index // 2, None)) + + _RECURSIVE_CUTOFF = 3 # Also hardcoded in traceback.c. + class StackSummary(list): - """A stack of frames.""" + """A list of FrameSummary objects, representing a stack of frames.""" @classmethod def extract(klass, frame_gen, *, limit=None, lookup_lines=True, capture_locals=False): """Create a StackSummary from a traceback or stack object. - :param frame_gen: A generator that yields (frame, lineno) tuples to - include in the stack. + :param frame_gen: A generator that yields (frame, lineno) tuples + whose summaries are to be included in the stack. :param limit: None to include all frames or the number of frames to include. :param lookup_lines: If True, lookup lines for each frame immediately, @@ -349,23 +442,41 @@ def extract(klass, frame_gen, *, limit=None, lookup_lines=True, :param capture_locals: If True, the local variables from each frame will be captured as object representations into the FrameSummary. """ - if limit is None: + def extended_frame_gen(): + for f, lineno in frame_gen: + yield f, (lineno, None, None, None) + + return klass._extract_from_extended_frame_gen( + extended_frame_gen(), limit=limit, lookup_lines=lookup_lines, + capture_locals=capture_locals) + + @classmethod + def _extract_from_extended_frame_gen(klass, frame_gen, *, limit=None, + lookup_lines=True, capture_locals=False): + # Same as extract but operates on a frame generator that yields + # (frame, (lineno, end_lineno, colno, end_colno)) in the stack. + # Only lineno is required, the remaining fields can be None if the + # information is not available. + builtin_limit = limit is BUILTIN_EXCEPTION_LIMIT + if limit is None or builtin_limit: limit = getattr(sys, 'tracebacklimit', None) if limit is not None and limit < 0: limit = 0 if limit is not None: - if limit >= 0: + if builtin_limit: + frame_gen = tuple(frame_gen) + frame_gen = frame_gen[len(frame_gen) - limit:] + elif limit >= 0: frame_gen = itertools.islice(frame_gen, limit) else: frame_gen = collections.deque(frame_gen, maxlen=-limit) result = klass() fnames = set() - for f, lineno in frame_gen: + for f, (lineno, end_lineno, colno, end_colno) in frame_gen: co = f.f_code filename = co.co_filename name = co.co_name - fnames.add(filename) linecache.lazycache(filename, f.f_globals) # Must defer line lookups until we have called checkcache. @@ -373,10 +484,16 @@ def extract(klass, frame_gen, *, limit=None, lookup_lines=True, f_locals = f.f_locals else: f_locals = None - result.append(FrameSummary( - filename, lineno, name, lookup_line=False, locals=f_locals)) + result.append( + FrameSummary(filename, lineno, name, + lookup_line=False, locals=f_locals, + end_lineno=end_lineno, colno=colno, end_colno=end_colno, + _code=f.f_code, + ) + ) for filename in fnames: linecache.checkcache(filename) + # If immediate lookup was desired, trigger lookups now. if lookup_lines: for f in result: @@ -402,7 +519,223 @@ def from_list(klass, a_list): result.append(FrameSummary(filename, lineno, name, line=line)) return result - def format(self): + def format_frame_summary(self, frame_summary, **kwargs): + """Format the lines for a single FrameSummary. + + Returns a string representing one frame involved in the stack. This + gets called for every frame to be printed in the stack summary. + """ + colorize = kwargs.get("colorize", False) + row = [] + filename = frame_summary.filename + if frame_summary.filename.startswith("'): + filename = "" + if colorize: + row.append(' File {}"{}"{}, line {}{}{}, in {}{}{}\n'.format( + ANSIColors.MAGENTA, + filename, + ANSIColors.RESET, + ANSIColors.MAGENTA, + frame_summary.lineno, + ANSIColors.RESET, + ANSIColors.MAGENTA, + frame_summary.name, + ANSIColors.RESET, + ) + ) + else: + row.append(' File "{}", line {}, in {}\n'.format( + filename, frame_summary.lineno, frame_summary.name)) + if frame_summary._dedented_lines and frame_summary._dedented_lines.strip(): + if ( + frame_summary.colno is None or + frame_summary.end_colno is None + ): + # only output first line if column information is missing + row.append(textwrap.indent(frame_summary.line, ' ') + "\n") + else: + # get first and last line + all_lines_original = frame_summary._original_lines.splitlines() + first_line = all_lines_original[0] + # assume all_lines_original has enough lines (since we constructed it) + last_line = all_lines_original[frame_summary.end_lineno - frame_summary.lineno] + + # character index of the start/end of the instruction + start_offset = _byte_offset_to_character_offset(first_line, frame_summary.colno) + end_offset = _byte_offset_to_character_offset(last_line, frame_summary.end_colno) + + all_lines = frame_summary._dedented_lines.splitlines()[ + :frame_summary.end_lineno - frame_summary.lineno + 1 + ] + + # adjust start/end offset based on dedent + dedent_characters = len(first_line) - len(all_lines[0]) + start_offset = max(0, start_offset - dedent_characters) + end_offset = max(0, end_offset - dedent_characters) + + # When showing this on a terminal, some of the non-ASCII characters + # might be rendered as double-width characters, so we need to take + # that into account when calculating the length of the line. + dp_start_offset = _display_width(all_lines[0], offset=start_offset) + dp_end_offset = _display_width(all_lines[-1], offset=end_offset) + + # get exact code segment corresponding to the instruction + segment = "\n".join(all_lines) + segment = segment[start_offset:len(segment) - (len(all_lines[-1]) - end_offset)] + + # attempt to parse for anchors + anchors = None + show_carets = False + with suppress(Exception): + anchors = _extract_caret_anchors_from_line_segment(segment) + show_carets = self._should_show_carets(start_offset, end_offset, all_lines, anchors) + + result = [] + + # only display first line, last line, and lines around anchor start/end + significant_lines = {0, len(all_lines) - 1} + + anchors_left_end_offset = 0 + anchors_right_start_offset = 0 + primary_char = "^" + secondary_char = "^" + if anchors: + anchors_left_end_offset = anchors.left_end_offset + anchors_right_start_offset = anchors.right_start_offset + # computed anchor positions do not take start_offset into account, + # so account for it here + if anchors.left_end_lineno == 0: + anchors_left_end_offset += start_offset + if anchors.right_start_lineno == 0: + anchors_right_start_offset += start_offset + + # account for display width + anchors_left_end_offset = _display_width( + all_lines[anchors.left_end_lineno], offset=anchors_left_end_offset + ) + anchors_right_start_offset = _display_width( + all_lines[anchors.right_start_lineno], offset=anchors_right_start_offset + ) + + primary_char = anchors.primary_char + secondary_char = anchors.secondary_char + significant_lines.update( + range(anchors.left_end_lineno - 1, anchors.left_end_lineno + 2) + ) + significant_lines.update( + range(anchors.right_start_lineno - 1, anchors.right_start_lineno + 2) + ) + + # remove bad line numbers + significant_lines.discard(-1) + significant_lines.discard(len(all_lines)) + + def output_line(lineno): + """output all_lines[lineno] along with carets""" + result.append(all_lines[lineno] + "\n") + if not show_carets: + return + num_spaces = len(all_lines[lineno]) - len(all_lines[lineno].lstrip()) + carets = [] + num_carets = dp_end_offset if lineno == len(all_lines) - 1 else _display_width(all_lines[lineno]) + # compute caret character for each position + for col in range(num_carets): + if col < num_spaces or (lineno == 0 and col < dp_start_offset): + # before first non-ws char of the line, or before start of instruction + carets.append(' ') + elif anchors and ( + lineno > anchors.left_end_lineno or + (lineno == anchors.left_end_lineno and col >= anchors_left_end_offset) + ) and ( + lineno < anchors.right_start_lineno or + (lineno == anchors.right_start_lineno and col < anchors_right_start_offset) + ): + # within anchors + carets.append(secondary_char) + else: + carets.append(primary_char) + if colorize: + # Replace the previous line with a red version of it only in the parts covered + # by the carets. + line = result[-1] + colorized_line_parts = [] + colorized_carets_parts = [] + + for color, group in itertools.groupby(itertools.zip_longest(line, carets, fillvalue=""), key=lambda x: x[1]): + caret_group = list(group) + if color == "^": + colorized_line_parts.append(ANSIColors.BOLD_RED + "".join(char for char, _ in caret_group) + ANSIColors.RESET) + colorized_carets_parts.append(ANSIColors.BOLD_RED + "".join(caret for _, caret in caret_group) + ANSIColors.RESET) + elif color == "~": + colorized_line_parts.append(ANSIColors.RED + "".join(char for char, _ in caret_group) + ANSIColors.RESET) + colorized_carets_parts.append(ANSIColors.RED + "".join(caret for _, caret in caret_group) + ANSIColors.RESET) + else: + colorized_line_parts.append("".join(char for char, _ in caret_group)) + colorized_carets_parts.append("".join(caret for _, caret in caret_group)) + + colorized_line = "".join(colorized_line_parts) + colorized_carets = "".join(colorized_carets_parts) + result[-1] = colorized_line + result.append(colorized_carets + "\n") + else: + result.append("".join(carets) + "\n") + + # display significant lines + sig_lines_list = sorted(significant_lines) + for i, lineno in enumerate(sig_lines_list): + if i: + linediff = lineno - sig_lines_list[i - 1] + if linediff == 2: + # 1 line in between - just output it + output_line(lineno - 1) + elif linediff > 2: + # > 1 line in between - abbreviate + result.append(f"...<{linediff - 1} lines>...\n") + output_line(lineno) + + row.append( + textwrap.indent(textwrap.dedent("".join(result)), ' ', lambda line: True) + ) + if frame_summary.locals: + for name, value in sorted(frame_summary.locals.items()): + row.append(' {name} = {value}\n'.format(name=name, value=value)) + + return ''.join(row) + + def _should_show_carets(self, start_offset, end_offset, all_lines, anchors): + with suppress(SyntaxError, ImportError): + import ast + tree = ast.parse('\n'.join(all_lines)) + if not tree.body: + return False + statement = tree.body[0] + value = None + def _spawns_full_line(value): + return ( + value.lineno == 1 + and value.end_lineno == len(all_lines) + and value.col_offset == start_offset + and value.end_col_offset == end_offset + ) + match statement: + case ast.Return(value=ast.Call()): + if isinstance(statement.value.func, ast.Name): + value = statement.value + case ast.Assign(value=ast.Call()): + if ( + len(statement.targets) == 1 and + isinstance(statement.targets[0], ast.Name) + ): + value = statement.value + if value is not None and _spawns_full_line(value): + return False + if anchors: + return True + if all_lines[0][:start_offset].lstrip() or all_lines[-1][end_offset:].rstrip(): + return True + return False + + def format(self, **kwargs): """Format the stack ready for printing. Returns a list of strings ready for printing. Each string in the @@ -414,37 +747,34 @@ def format(self): repetitions are shown, followed by a summary line stating the exact number of further repetitions. """ + colorize = kwargs.get("colorize", False) result = [] last_file = None last_line = None last_name = None count = 0 - for frame in self: - if (last_file is None or last_file != frame.filename or - last_line is None or last_line != frame.lineno or - last_name is None or last_name != frame.name): + for frame_summary in self: + formatted_frame = self.format_frame_summary(frame_summary, colorize=colorize) + if formatted_frame is None: + continue + if (last_file is None or last_file != frame_summary.filename or + last_line is None or last_line != frame_summary.lineno or + last_name is None or last_name != frame_summary.name): if count > _RECURSIVE_CUTOFF: count -= _RECURSIVE_CUTOFF result.append( f' [Previous line repeated {count} more ' f'time{"s" if count > 1 else ""}]\n' ) - last_file = frame.filename - last_line = frame.lineno - last_name = frame.name + last_file = frame_summary.filename + last_line = frame_summary.lineno + last_name = frame_summary.name count = 0 count += 1 if count > _RECURSIVE_CUTOFF: continue - row = [] - row.append(' File "{}", line {}, in {}\n'.format( - frame.filename, frame.lineno, frame.name)) - if frame.line: - row.append(' {}\n'.format(frame.line.strip())) - if frame.locals: - for name, value in sorted(frame.locals.items()): - row.append(' {name} = {value}\n'.format(name=name, value=value)) - result.append(''.join(row)) + result.append(formatted_frame) + if count > _RECURSIVE_CUTOFF: count -= _RECURSIVE_CUTOFF result.append( @@ -454,6 +784,216 @@ def format(self): return result +def _byte_offset_to_character_offset(str, offset): + as_utf8 = str.encode('utf-8') + return len(as_utf8[:offset].decode("utf-8", errors="replace")) + + +_Anchors = collections.namedtuple( + "_Anchors", + [ + "left_end_lineno", + "left_end_offset", + "right_start_lineno", + "right_start_offset", + "primary_char", + "secondary_char", + ], + defaults=["~", "^"] +) + +def _extract_caret_anchors_from_line_segment(segment): + """ + Given source code `segment` corresponding to a FrameSummary, determine: + - for binary ops, the location of the binary op + - for indexing and function calls, the location of the brackets. + `segment` is expected to be a valid Python expression. + """ + import ast + + try: + # Without parentheses, `segment` is parsed as a statement. + # Binary ops, subscripts, and calls are expressions, so + # we can wrap them with parentheses to parse them as + # (possibly multi-line) expressions. + # e.g. if we try to highlight the addition in + # x = ( + # a + + # b + # ) + # then we would ast.parse + # a + + # b + # which is not a valid statement because of the newline. + # Adding brackets makes it a valid expression. + # ( + # a + + # b + # ) + # Line locations will be different than the original, + # which is taken into account later on. + tree = ast.parse(f"(\n{segment}\n)") + except SyntaxError: + return None + + if len(tree.body) != 1: + return None + + lines = segment.splitlines() + + def normalize(lineno, offset): + """Get character index given byte offset""" + return _byte_offset_to_character_offset(lines[lineno], offset) + + def next_valid_char(lineno, col): + """Gets the next valid character index in `lines`, if + the current location is not valid. Handles empty lines. + """ + while lineno < len(lines) and col >= len(lines[lineno]): + col = 0 + lineno += 1 + assert lineno < len(lines) and col < len(lines[lineno]) + return lineno, col + + def increment(lineno, col): + """Get the next valid character index in `lines`.""" + col += 1 + lineno, col = next_valid_char(lineno, col) + return lineno, col + + def nextline(lineno, col): + """Get the next valid character at least on the next line""" + col = 0 + lineno += 1 + lineno, col = next_valid_char(lineno, col) + return lineno, col + + def increment_until(lineno, col, stop): + """Get the next valid non-"\\#" character that satisfies the `stop` predicate""" + while True: + ch = lines[lineno][col] + if ch in "\\#": + lineno, col = nextline(lineno, col) + elif not stop(ch): + lineno, col = increment(lineno, col) + else: + break + return lineno, col + + def setup_positions(expr, force_valid=True): + """Get the lineno/col position of the end of `expr`. If `force_valid` is True, + forces the position to be a valid character (e.g. if the position is beyond the + end of the line, move to the next line) + """ + # -2 since end_lineno is 1-indexed and because we added an extra + # bracket + newline to `segment` when calling ast.parse + lineno = expr.end_lineno - 2 + col = normalize(lineno, expr.end_col_offset) + return next_valid_char(lineno, col) if force_valid else (lineno, col) + + statement = tree.body[0] + match statement: + case ast.Expr(expr): + match expr: + case ast.BinOp(): + # ast gives these locations for BinOp subexpressions + # ( left_expr ) + ( right_expr ) + # left^^^^^ right^^^^^ + lineno, col = setup_positions(expr.left) + + # First operator character is the first non-space/')' character + lineno, col = increment_until(lineno, col, lambda x: not x.isspace() and x != ')') + + # binary op is 1 or 2 characters long, on the same line, + # before the right subexpression + right_col = col + 1 + if ( + right_col < len(lines[lineno]) + and ( + # operator char should not be in the right subexpression + expr.right.lineno - 2 > lineno or + right_col < normalize(expr.right.lineno - 2, expr.right.col_offset) + ) + and not (ch := lines[lineno][right_col]).isspace() + and ch not in "\\#" + ): + right_col += 1 + + # right_col can be invalid since it is exclusive + return _Anchors(lineno, col, lineno, right_col) + case ast.Subscript(): + # ast gives these locations for value and slice subexpressions + # ( value_expr ) [ slice_expr ] + # value^^^^^ slice^^^^^ + # subscript^^^^^^^^^^^^^^^^^^^^ + + # find left bracket + left_lineno, left_col = setup_positions(expr.value) + left_lineno, left_col = increment_until(left_lineno, left_col, lambda x: x == '[') + # find right bracket (final character of expression) + right_lineno, right_col = setup_positions(expr, force_valid=False) + return _Anchors(left_lineno, left_col, right_lineno, right_col) + case ast.Call(): + # ast gives these locations for function call expressions + # ( func_expr ) (args, kwargs) + # func^^^^^ + # call^^^^^^^^^^^^^^^^^^^^^^^^ + + # find left bracket + left_lineno, left_col = setup_positions(expr.func) + left_lineno, left_col = increment_until(left_lineno, left_col, lambda x: x == '(') + # find right bracket (final character of expression) + right_lineno, right_col = setup_positions(expr, force_valid=False) + return _Anchors(left_lineno, left_col, right_lineno, right_col) + + return None + +_WIDE_CHAR_SPECIFIERS = "WF" + +def _display_width(line, offset=None): + """Calculate the extra amount of width space the given source + code segment might take if it were to be displayed on a fixed + width output device. Supports wide unicode characters and emojis.""" + + if offset is None: + offset = len(line) + + # Fast track for ASCII-only strings + if line.isascii(): + return offset + + import unicodedata + + return sum( + 2 if unicodedata.east_asian_width(char) in _WIDE_CHAR_SPECIFIERS else 1 + for char in line[:offset] + ) + + + +class _ExceptionPrintContext: + def __init__(self): + self.seen = set() + self.exception_group_depth = 0 + self.need_close = False + + def indent(self): + return ' ' * (2 * self.exception_group_depth) + + def emit(self, text_gen, margin_char=None): + if margin_char is None: + margin_char = '|' + indent_str = self.indent() + if self.exception_group_depth: + indent_str += margin_char + ' ' + + if isinstance(text_gen, str): + yield textwrap.indent(text_gen, indent_str, lambda line: True) + else: + for text in text_gen: + yield textwrap.indent(text, indent_str, lambda line: True) + + class TracebackException: """An exception ready for rendering. @@ -461,16 +1001,24 @@ class TracebackException: to this intermediary form to ensure that no references are held, while still being able to fully print or format it. + max_group_width and max_group_depth control the formatting of exception + groups. The depth refers to the nesting level of the group, and the width + refers to the size of a single exception group's exceptions array. The + formatted output is truncated when either limit is exceeded. + Use `from_exception` to create TracebackException instances from exception objects, or the constructor to create TracebackException instances from individual components. - :attr:`__cause__` A TracebackException of the original *__cause__*. - :attr:`__context__` A TracebackException of the original *__context__*. + - :attr:`exceptions` For exception groups - a list of TracebackException + instances for the nested *exceptions*. ``None`` for other exceptions. - :attr:`__suppress_context__` The *__suppress_context__* value from the original exception. - :attr:`stack` A `StackSummary` representing the traceback. - - :attr:`exc_type` The class of the original traceback. + - :attr:`exc_type` (deprecated) The class of the original traceback. + - :attr:`exc_type_str` String display of exc_type - :attr:`filename` For syntax errors - the filename where the error occurred. - :attr:`lineno` For syntax errors - the linenumber where the error @@ -481,14 +1029,14 @@ class TracebackException: occurred. - :attr:`offset` For syntax errors - the offset into the text where the error occurred. - - :attr:`end_offset` For syntax errors - the offset into the text where the - error occurred. Can be `None` if not present. + - :attr:`end_offset` For syntax errors - the end offset into the text where + the error occurred. Can be `None` if not present. - :attr:`msg` For syntax errors - the compiler error message. """ def __init__(self, exc_type, exc_value, exc_traceback, *, limit=None, lookup_lines=True, capture_locals=False, compact=False, - _seen=None): + max_group_width=15, max_group_depth=10, save_exc_type=True, _seen=None): # NB: we need to accept exc_traceback, exc_value, exc_traceback to # permit backwards compat with the existing API, otherwise we # need stub thunk objects just to glue it together. @@ -498,14 +1046,34 @@ def __init__(self, exc_type, exc_value, exc_traceback, *, limit=None, _seen = set() _seen.add(id(exc_value)) - # TODO: locals. - self.stack = StackSummary.extract( - walk_tb(exc_traceback), limit=limit, lookup_lines=lookup_lines, + self.max_group_width = max_group_width + self.max_group_depth = max_group_depth + + self.stack = StackSummary._extract_from_extended_frame_gen( + _walk_tb_with_full_positions(exc_traceback), + limit=limit, lookup_lines=lookup_lines, capture_locals=capture_locals) - self.exc_type = exc_type + + self._exc_type = exc_type if save_exc_type else None + # Capture now to permit freeing resources: only complication is in the # unofficial API _format_final_exc_line - self._str = _some_str(exc_value) + self._str = _safe_string(exc_value, 'exception') + try: + self.__notes__ = getattr(exc_value, '__notes__', None) + except Exception as e: + self.__notes__ = [ + f'Ignored error getting __notes__: {_safe_string(e, '__notes__', repr)}'] + + self._is_syntax_error = False + self._have_exc_type = exc_type is not None + if exc_type is not None: + self.exc_type_qualname = exc_type.__qualname__ + self.exc_type_module = exc_type.__module__ + else: + self.exc_type_qualname = None + self.exc_type_module = None + if exc_type and issubclass(exc_type, SyntaxError): # Handle SyntaxError's specially self.filename = exc_value.filename @@ -517,6 +1085,26 @@ def __init__(self, exc_type, exc_value, exc_traceback, *, limit=None, self.offset = exc_value.offset self.end_offset = exc_value.end_offset self.msg = exc_value.msg + self._is_syntax_error = True + elif exc_type and issubclass(exc_type, ImportError) and \ + getattr(exc_value, "name_from", None) is not None: + wrong_name = getattr(exc_value, "name_from", None) + suggestion = _compute_suggestion_error(exc_value, exc_traceback, wrong_name) + if suggestion: + self._str += f". Did you mean: '{suggestion}'?" + elif exc_type and issubclass(exc_type, (NameError, AttributeError)) and \ + getattr(exc_value, "name", None) is not None: + wrong_name = getattr(exc_value, "name", None) + suggestion = _compute_suggestion_error(exc_value, exc_traceback, wrong_name) + if suggestion: + self._str += f". Did you mean: '{suggestion}'?" + if issubclass(exc_type, NameError): + wrong_name = getattr(exc_value, "name", None) + if wrong_name is not None and wrong_name in sys.stdlib_module_names: + if suggestion: + self._str += f" Or did you forget to import '{wrong_name}'?" + else: + self._str += f". Did you forget to import '{wrong_name}'?" if lookup_lines: self._load_lines() self.__suppress_context__ = \ @@ -528,7 +1116,7 @@ def __init__(self, exc_type, exc_value, exc_traceback, *, limit=None, queue = [(self, exc_value)] while queue: te, e = queue.pop() - if (e and e.__cause__ is not None + if (e is not None and e.__cause__ is not None and id(e.__cause__) not in _seen): cause = TracebackException( type(e.__cause__), @@ -537,6 +1125,8 @@ def __init__(self, exc_type, exc_value, exc_traceback, *, limit=None, limit=limit, lookup_lines=lookup_lines, capture_locals=capture_locals, + max_group_width=max_group_width, + max_group_depth=max_group_depth, _seen=_seen) else: cause = None @@ -547,7 +1137,7 @@ def __init__(self, exc_type, exc_value, exc_traceback, *, limit=None, not e.__suppress_context__) else: need_context = True - if (e and e.__context__ is not None + if (e is not None and e.__context__ is not None and need_context and id(e.__context__) not in _seen): context = TracebackException( type(e.__context__), @@ -556,21 +1146,62 @@ def __init__(self, exc_type, exc_value, exc_traceback, *, limit=None, limit=limit, lookup_lines=lookup_lines, capture_locals=capture_locals, + max_group_width=max_group_width, + max_group_depth=max_group_depth, _seen=_seen) else: context = None + + if e is not None and isinstance(e, BaseExceptionGroup): + exceptions = [] + for exc in e.exceptions: + texc = TracebackException( + type(exc), + exc, + exc.__traceback__, + limit=limit, + lookup_lines=lookup_lines, + capture_locals=capture_locals, + max_group_width=max_group_width, + max_group_depth=max_group_depth, + _seen=_seen) + exceptions.append(texc) + else: + exceptions = None + te.__cause__ = cause te.__context__ = context + te.exceptions = exceptions if cause: queue.append((te.__cause__, e.__cause__)) if context: queue.append((te.__context__, e.__context__)) + if exceptions: + queue.extend(zip(te.exceptions, e.exceptions)) @classmethod def from_exception(cls, exc, *args, **kwargs): """Create a TracebackException from an exception.""" return cls(type(exc), exc, exc.__traceback__, *args, **kwargs) + @property + def exc_type(self): + warnings.warn('Deprecated in 3.13. Use exc_type_str instead.', + DeprecationWarning, stacklevel=2) + return self._exc_type + + @property + def exc_type_str(self): + if not self._have_exc_type: + return None + stype = self.exc_type_qualname + smod = self.exc_type_module + if smod not in ("__main__", "builtins"): + if not isinstance(smod, str): + smod = "" + stype = smod + '.' + stype + return stype + def _load_lines(self): """Private API. force all lines in the stack to be loaded.""" for frame in self.stack: @@ -584,72 +1215,152 @@ def __eq__(self, other): def __str__(self): return self._str - def format_exception_only(self): + def format_exception_only(self, *, show_group=False, _depth=0, **kwargs): """Format the exception part of the traceback. The return value is a generator of strings, each ending in a newline. - Normally, the generator emits a single string; however, for - SyntaxError exceptions, it emits several lines that (when - printed) display detailed information about where the syntax - error occurred. - - The message indicating which exception occurred is always the last - string in the output. + Generator yields the exception message. + For :exc:`SyntaxError` exceptions, it + also yields (before the exception message) + several lines that (when printed) + display detailed information about where the syntax error occurred. + Following the message, generator also yields + all the exception's ``__notes__``. + + When *show_group* is ``True``, and the exception is an instance of + :exc:`BaseExceptionGroup`, the nested exceptions are included as + well, recursively, with indentation relative to their nesting depth. """ - if self.exc_type is None: - yield _format_final_exc_line(None, self._str) - return + colorize = kwargs.get("colorize", False) - stype = self.exc_type.__qualname__ - smod = self.exc_type.__module__ - if smod not in ("__main__", "builtins"): - if not isinstance(smod, str): - smod = "" - stype = smod + '.' + stype + indent = 3 * _depth * ' ' + if not self._have_exc_type: + yield indent + _format_final_exc_line(None, self._str, colorize=colorize) + return - if not issubclass(self.exc_type, SyntaxError): - yield _format_final_exc_line(stype, self._str) + stype = self.exc_type_str + if not self._is_syntax_error: + if _depth > 0: + # Nested exceptions needs correct handling of multiline messages. + formatted = _format_final_exc_line( + stype, self._str, insert_final_newline=False, colorize=colorize + ).split('\n') + yield from [ + indent + l + '\n' + for l in formatted + ] + else: + yield _format_final_exc_line(stype, self._str, colorize=colorize) else: - yield from self._format_syntax_error(stype) - - def _format_syntax_error(self, stype): + yield from [indent + l for l in self._format_syntax_error(stype, colorize=colorize)] + + if ( + isinstance(self.__notes__, collections.abc.Sequence) + and not isinstance(self.__notes__, (str, bytes)) + ): + for note in self.__notes__: + note = _safe_string(note, 'note') + yield from [indent + l + '\n' for l in note.split('\n')] + elif self.__notes__ is not None: + yield indent + "{}\n".format(_safe_string(self.__notes__, '__notes__', func=repr)) + + if self.exceptions and show_group: + for ex in self.exceptions: + yield from ex.format_exception_only(show_group=show_group, _depth=_depth+1, colorize=colorize) + + def _format_syntax_error(self, stype, **kwargs): """Format SyntaxError exceptions (internal helper).""" # Show exactly where the problem was found. + colorize = kwargs.get("colorize", False) filename_suffix = '' if self.lineno is not None: - yield ' File "{}", line {}\n'.format( - self.filename or "", self.lineno) + if colorize: + yield ' File {}"{}"{}, line {}{}{}\n'.format( + ANSIColors.MAGENTA, + self.filename or "", + ANSIColors.RESET, + ANSIColors.MAGENTA, + self.lineno, + ANSIColors.RESET, + ) + else: + yield ' File "{}", line {}\n'.format( + self.filename or "", self.lineno) elif self.filename is not None: filename_suffix = ' ({})'.format(self.filename) text = self.text - if text is not None: + if isinstance(text, str): # text = " foo\n" # rtext = " foo" # ltext = "foo" rtext = text.rstrip('\n') ltext = rtext.lstrip(' \n\f') spaces = len(rtext) - len(ltext) - yield ' {}\n'.format(ltext) - - if self.offset is not None: + if self.offset is None: + yield ' {}\n'.format(ltext) + elif isinstance(self.offset, int): offset = self.offset - end_offset = self.end_offset if self.end_offset not in {None, 0} else offset - if offset == end_offset or end_offset == -1: + if self.lineno == self.end_lineno: + end_offset = ( + self.end_offset + if ( + isinstance(self.end_offset, int) + and self.end_offset != 0 + ) + else offset + ) + else: + end_offset = len(rtext) + 1 + + if self.text and offset > len(self.text): + offset = len(rtext) + 1 + if self.text and end_offset > len(self.text): + end_offset = len(rtext) + 1 + if offset >= end_offset or end_offset < 0: end_offset = offset + 1 # Convert 1-based column offset to 0-based index into stripped text colno = offset - 1 - spaces end_colno = end_offset - 1 - spaces + caretspace = ' ' if colno >= 0: # non-space whitespace (likes tabs) must be kept for alignment caretspace = ((c if c.isspace() else ' ') for c in ltext[:colno]) - yield ' {}{}'.format("".join(caretspace), ('^' * (end_colno - colno) + "\n")) + start_color = end_color = "" + if colorize: + # colorize from colno to end_colno + ltext = ( + ltext[:colno] + + ANSIColors.BOLD_RED + ltext[colno:end_colno] + ANSIColors.RESET + + ltext[end_colno:] + ) + start_color = ANSIColors.BOLD_RED + end_color = ANSIColors.RESET + yield ' {}\n'.format(ltext) + yield ' {}{}{}{}\n'.format( + "".join(caretspace), + start_color, + ('^' * (end_colno - colno)), + end_color, + ) + else: + yield ' {}\n'.format(ltext) msg = self.msg or "" - yield "{}: {}{}\n".format(stype, msg, filename_suffix) + if colorize: + yield "{}{}{}: {}{}{}{}\n".format( + ANSIColors.BOLD_MAGENTA, + stype, + ANSIColors.RESET, + ANSIColors.MAGENTA, + msg, + ANSIColors.RESET, + filename_suffix) + else: + yield "{}: {}{}\n".format(stype, msg, filename_suffix) - def format(self, *, chain=True): + def format(self, *, chain=True, _ctx=None, **kwargs): """Format the exception. If chain is not *True*, *__cause__* and *__context__* will not be formatted. @@ -661,11 +1372,14 @@ def format(self, *, chain=True): The message indicating which exception occurred is always the last string in the output. """ + colorize = kwargs.get("colorize", False) + if _ctx is None: + _ctx = _ExceptionPrintContext() output = [] exc = self - while exc: - if chain: + if chain: + while exc: if exc.__cause__ is not None: chained_msg = _cause_message chained_exc = exc.__cause__ @@ -679,14 +1393,246 @@ def format(self, *, chain=True): output.append((chained_msg, exc)) exc = chained_exc - else: - output.append((None, exc)) - exc = None + else: + output.append((None, exc)) for msg, exc in reversed(output): if msg is not None: - yield msg - if exc.stack: - yield 'Traceback (most recent call last):\n' - yield from exc.stack.format() - yield from exc.format_exception_only() + yield from _ctx.emit(msg) + if exc.exceptions is None: + if exc.stack: + yield from _ctx.emit('Traceback (most recent call last):\n') + yield from _ctx.emit(exc.stack.format(colorize=colorize)) + yield from _ctx.emit(exc.format_exception_only(colorize=colorize)) + elif _ctx.exception_group_depth > self.max_group_depth: + # exception group, but depth exceeds limit + yield from _ctx.emit( + f"... (max_group_depth is {self.max_group_depth})\n") + else: + # format exception group + is_toplevel = (_ctx.exception_group_depth == 0) + if is_toplevel: + _ctx.exception_group_depth += 1 + + if exc.stack: + yield from _ctx.emit( + 'Exception Group Traceback (most recent call last):\n', + margin_char = '+' if is_toplevel else None) + yield from _ctx.emit(exc.stack.format(colorize=colorize)) + + yield from _ctx.emit(exc.format_exception_only(colorize=colorize)) + num_excs = len(exc.exceptions) + if num_excs <= self.max_group_width: + n = num_excs + else: + n = self.max_group_width + 1 + _ctx.need_close = False + for i in range(n): + last_exc = (i == n-1) + if last_exc: + # The closing frame may be added by a recursive call + _ctx.need_close = True + + if self.max_group_width is not None: + truncated = (i >= self.max_group_width) + else: + truncated = False + title = f'{i+1}' if not truncated else '...' + yield (_ctx.indent() + + ('+-' if i==0 else ' ') + + f'+---------------- {title} ----------------\n') + _ctx.exception_group_depth += 1 + if not truncated: + yield from exc.exceptions[i].format(chain=chain, _ctx=_ctx, colorize=colorize) + else: + remaining = num_excs - self.max_group_width + plural = 's' if remaining > 1 else '' + yield from _ctx.emit( + f"and {remaining} more exception{plural}\n") + + if last_exc and _ctx.need_close: + yield (_ctx.indent() + + "+------------------------------------\n") + _ctx.need_close = False + _ctx.exception_group_depth -= 1 + + if is_toplevel: + assert _ctx.exception_group_depth == 1 + _ctx.exception_group_depth = 0 + + + def print(self, *, file=None, chain=True, **kwargs): + """Print the result of self.format(chain=chain) to 'file'.""" + colorize = kwargs.get("colorize", False) + if file is None: + file = sys.stderr + for line in self.format(chain=chain, colorize=colorize): + print(line, file=file, end="") + + +_MAX_CANDIDATE_ITEMS = 750 +_MAX_STRING_SIZE = 40 +_MOVE_COST = 2 +_CASE_COST = 1 + + +def _substitution_cost(ch_a, ch_b): + if ch_a == ch_b: + return 0 + if ch_a.lower() == ch_b.lower(): + return _CASE_COST + return _MOVE_COST + + +def _compute_suggestion_error(exc_value, tb, wrong_name): + if wrong_name is None or not isinstance(wrong_name, str): + return None + if isinstance(exc_value, AttributeError): + obj = exc_value.obj + try: + try: + d = dir(obj) + except TypeError: # Attributes are unsortable, e.g. int and str + d = list(obj.__class__.__dict__.keys()) + list(obj.__dict__.keys()) + d = sorted([x for x in d if isinstance(x, str)]) + hide_underscored = (wrong_name[:1] != '_') + if hide_underscored and tb is not None: + while tb.tb_next is not None: + tb = tb.tb_next + frame = tb.tb_frame + if 'self' in frame.f_locals and frame.f_locals['self'] is obj: + hide_underscored = False + if hide_underscored: + d = [x for x in d if x[:1] != '_'] + except Exception: + return None + elif isinstance(exc_value, ImportError): + try: + mod = __import__(exc_value.name) + try: + d = dir(mod) + except TypeError: # Attributes are unsortable, e.g. int and str + d = list(mod.__dict__.keys()) + d = sorted([x for x in d if isinstance(x, str)]) + if wrong_name[:1] != '_': + d = [x for x in d if x[:1] != '_'] + except Exception: + return None + else: + assert isinstance(exc_value, NameError) + # find most recent frame + if tb is None: + return None + while tb.tb_next is not None: + tb = tb.tb_next + frame = tb.tb_frame + d = ( + list(frame.f_locals) + + list(frame.f_globals) + + list(frame.f_builtins) + ) + d = [x for x in d if isinstance(x, str)] + + # Check first if we are in a method and the instance + # has the wrong name as attribute + if 'self' in frame.f_locals: + self = frame.f_locals['self'] + try: + has_wrong_name = hasattr(self, wrong_name) + except Exception: + has_wrong_name = False + if has_wrong_name: + return f"self.{wrong_name}" + + try: + import _suggestions + except ImportError: + pass + else: + return _suggestions._generate_suggestions(d, wrong_name) + + # Compute closest match + + if len(d) > _MAX_CANDIDATE_ITEMS: + return None + wrong_name_len = len(wrong_name) + if wrong_name_len > _MAX_STRING_SIZE: + return None + best_distance = wrong_name_len + suggestion = None + for possible_name in d: + if possible_name == wrong_name: + # A missing attribute is "found". Don't suggest it (see GH-88821). + continue + # No more than 1/3 of the involved characters should need changed. + max_distance = (len(possible_name) + wrong_name_len + 3) * _MOVE_COST // 6 + # Don't take matches we've already beaten. + max_distance = min(max_distance, best_distance - 1) + current_distance = _levenshtein_distance(wrong_name, possible_name, max_distance) + if current_distance > max_distance: + continue + if not suggestion or current_distance < best_distance: + suggestion = possible_name + best_distance = current_distance + return suggestion + + +def _levenshtein_distance(a, b, max_cost): + # A Python implementation of Python/suggestions.c:levenshtein_distance. + + # Both strings are the same + if a == b: + return 0 + + # Trim away common affixes + pre = 0 + while a[pre:] and b[pre:] and a[pre] == b[pre]: + pre += 1 + a = a[pre:] + b = b[pre:] + post = 0 + while a[:post or None] and b[:post or None] and a[post-1] == b[post-1]: + post -= 1 + a = a[:post or None] + b = b[:post or None] + if not a or not b: + return _MOVE_COST * (len(a) + len(b)) + if len(a) > _MAX_STRING_SIZE or len(b) > _MAX_STRING_SIZE: + return max_cost + 1 + + # Prefer shorter buffer + if len(b) < len(a): + a, b = b, a + + # Quick fail when a match is impossible + if (len(b) - len(a)) * _MOVE_COST > max_cost: + return max_cost + 1 + + # Instead of producing the whole traditional len(a)-by-len(b) + # matrix, we can update just one row in place. + # Initialize the buffer row + row = list(range(_MOVE_COST, _MOVE_COST * (len(a) + 1), _MOVE_COST)) + + result = 0 + for bindex in range(len(b)): + bchar = b[bindex] + distance = result = bindex * _MOVE_COST + minimum = sys.maxsize + for index in range(len(a)): + # 1) Previous distance in this row is cost(b[:b_index], a[:index]) + substitute = distance + _substitution_cost(bchar, a[index]) + # 2) cost(b[:b_index], a[:index+1]) from previous row + distance = row[index] + # 3) existing result is cost(b[:b_index+1], a[index]) + + insert_delete = min(result, distance) + _MOVE_COST + result = min(insert_delete, substitute) + + # cost(b[:b_index+1], a[:index+1]) + row[index] = result + if result < minimum: + minimum = result + if minimum > max_cost: + # Everything in this row is too big, so bail early. + return max_cost + 1 + return result diff --git a/crates/codegen/src/compile.rs b/crates/codegen/src/compile.rs index c44b2b00684..5cc7d0d2212 100644 --- a/crates/codegen/src/compile.rs +++ b/crates/codegen/src/compile.rs @@ -115,11 +115,22 @@ enum DoneWithFuture { Yes, } -#[derive(Debug, Clone, Default)] +#[derive(Debug, Clone)] pub struct CompileOpts { /// How optimized the bytecode output should be; any optimize > 0 does /// not emit assert statements pub optimize: u8, + /// Include column info in bytecode (-X no_debug_ranges disables) + pub debug_ranges: bool, +} + +impl Default for CompileOpts { + fn default() -> Self { + Self { + optimize: 0, + debug_ranges: true, + } + } } #[derive(Debug, Clone, Copy)] @@ -859,7 +870,7 @@ impl Compiler { let pop = self.code_stack.pop(); let stack_top = compiler_unwrap_option(self, pop); // No parent scope stack to maintain - unwrap_internal(self, stack_top.finalize_code(self.opts.optimize)) + unwrap_internal(self, stack_top.finalize_code(&self.opts)) } /// Push a new fblock @@ -1486,7 +1497,9 @@ impl Compiler { .. }) => self.compile_for(target, iter, body, orelse, *is_async)?, Stmt::Match(StmtMatch { subject, cases, .. }) => self.compile_match(subject, cases)?, - Stmt::Raise(StmtRaise { exc, cause, .. }) => { + Stmt::Raise(StmtRaise { + exc, cause, range, .. + }) => { let kind = match exc { Some(value) => { self.compile_expression(value)?; @@ -1500,6 +1513,7 @@ impl Compiler { } None => bytecode::RaiseKind::Reraise, }; + self.set_source_range(*range); emit!(self, Instruction::Raise { kind }); } Stmt::Try(StmtTry { @@ -5639,17 +5653,15 @@ impl Compiler { // Low level helper functions: fn _emit(&mut self, instr: Instruction, arg: OpArg, target: BlockIdx) { let range = self.current_source_range; - let location = self - .source_file - .to_source_code() - .source_location(range.start(), PositionEncoding::Utf8); - // TODO: insert source filename + let source = self.source_file.to_source_code(); + let location = source.source_location(range.start(), PositionEncoding::Utf8); + let end_location = source.source_location(range.end(), PositionEncoding::Utf8); self.current_block().instructions.push(ir::InstructionInfo { instr, arg, target, location, - // range, + end_location, }); } diff --git a/crates/codegen/src/ir.rs b/crates/codegen/src/ir.rs index 670635fbd37..31ee8d8b230 100644 --- a/crates/codegen/src/ir.rs +++ b/crates/codegen/src/ir.rs @@ -86,9 +86,8 @@ pub struct InstructionInfo { pub instr: Instruction, pub arg: OpArg, pub target: BlockIdx, - // pub range: TextRange, pub location: SourceLocation, - // TODO: end_location for debug ranges + pub end_location: SourceLocation, } // spell-checker:ignore petgraph @@ -133,8 +132,11 @@ pub struct CodeInfo { } impl CodeInfo { - pub fn finalize_code(mut self, optimize: u8) -> crate::InternalResult { - if optimize > 0 { + pub fn finalize_code( + mut self, + opts: &crate::compile::CompileOpts, + ) -> crate::InternalResult { + if opts.optimize > 0 { self.dce(); } @@ -198,7 +200,10 @@ impl CodeInfo { *arg = new_arg; } let (extras, lo_arg) = arg.split(); - locations.extend(core::iter::repeat_n(info.location, arg.instr_size())); + locations.extend(core::iter::repeat_n( + (info.location, info.end_location), + arg.instr_size(), + )); instructions.extend( extras .map(|byte| CodeUnit::new(Instruction::ExtendedArg, byte)) @@ -217,7 +222,11 @@ impl CodeInfo { } // Generate linetable from locations - let linetable = generate_linetable(&locations, first_line_number.get() as i32); + let linetable = generate_linetable( + &locations, + first_line_number.get() as i32, + opts.debug_ranges, + ); Ok(CodeObject { flags, @@ -412,7 +421,11 @@ fn iter_blocks(blocks: &[Block]) -> impl Iterator + ' } /// Generate CPython 3.11+ format linetable from source locations -fn generate_linetable(locations: &[SourceLocation], first_line: i32) -> Box<[u8]> { +fn generate_linetable( + locations: &[(SourceLocation, SourceLocation)], + first_line: i32, + debug_ranges: bool, +) -> Box<[u8]> { if locations.is_empty() { return Box::new([]); } @@ -424,7 +437,7 @@ fn generate_linetable(locations: &[SourceLocation], first_line: i32) -> Box<[u8] let mut i = 0; while i < locations.len() { - let loc = &locations[i]; + let (loc, end_loc) = &locations[i]; // Count consecutive instructions with the same location let mut length = 1; @@ -436,18 +449,33 @@ fn generate_linetable(locations: &[SourceLocation], first_line: i32) -> Box<[u8] while length > 0 { let entry_length = length.min(8); - // Get line and column information - // SourceLocation always has row and column (both are OneIndexed) + // Get line information let line = loc.line.get() as i32; - let col = loc.character_offset.to_zero_indexed() as i32; - + let end_line = end_loc.line.get() as i32; let line_delta = line - prev_line; + let end_line_delta = end_line - line; - // Choose the appropriate encoding based on line delta and column info - // Note: SourceLocation always has valid column, so we never get NO_COLUMNS case - if line_delta == 0 { - let end_col = col; // Use same column for end (no range info available) + // When debug_ranges is disabled, only emit line info (NoColumns format) + if !debug_ranges { + // NoColumns format (code 13): line info only, no column data + linetable.push( + 0x80 | ((PyCodeLocationInfoKind::NoColumns as u8) << 3) + | ((entry_length - 1) as u8), + ); + write_signed_varint(&mut linetable, line_delta); + prev_line = line; + length -= entry_length; + i += entry_length; + continue; + } + + // Get column information (only when debug_ranges is enabled) + let col = loc.character_offset.to_zero_indexed() as i32; + let end_col = end_loc.character_offset.to_zero_indexed() as i32; + + // Choose the appropriate encoding based on line delta and column info + if line_delta == 0 && end_line_delta == 0 { if col < 80 && end_col - col < 16 && end_col >= col { // Short form (codes 0-9) for common cases let code = (col / 8).min(9) as u8; // Short0 to Short9 @@ -470,42 +498,37 @@ fn generate_linetable(locations: &[SourceLocation], first_line: i32) -> Box<[u8] ); write_signed_varint(&mut linetable, 0); // line_delta = 0 write_varint(&mut linetable, 0); // end_line delta = 0 - write_varint(&mut linetable, (col as u32) + 1); // column + 1 for encoding - write_varint(&mut linetable, (end_col as u32) + 1); // end_col + 1 + write_varint(&mut linetable, (col as u32) + 1); + write_varint(&mut linetable, (end_col as u32) + 1); } - } else if line_delta > 0 && line_delta < 3 - /* && column.is_some() */ - { + } else if line_delta > 0 && line_delta < 3 && end_line_delta == 0 { // One-line form (codes 11-12) for line deltas 1-2 - let end_col = col; // Use same column for end - if col < 128 && end_col < 128 { - let code = (PyCodeLocationInfoKind::OneLine0 as u8) + (line_delta as u8); // 11 for delta=1, 12 for delta=2 + let code = (PyCodeLocationInfoKind::OneLine0 as u8) + (line_delta as u8); linetable.push(0x80 | (code << 3) | ((entry_length - 1) as u8)); linetable.push(col as u8); linetable.push(end_col as u8); } else { - // Long form for columns >= 128 or negative line delta + // Long form for columns >= 128 linetable.push( 0x80 | ((PyCodeLocationInfoKind::Long as u8) << 3) | ((entry_length - 1) as u8), ); write_signed_varint(&mut linetable, line_delta); write_varint(&mut linetable, 0); // end_line delta = 0 - write_varint(&mut linetable, (col as u32) + 1); // column + 1 for encoding - write_varint(&mut linetable, (end_col as u32) + 1); // end_col + 1 + write_varint(&mut linetable, (col as u32) + 1); + write_varint(&mut linetable, (end_col as u32) + 1); } } else { // Long form (code 14) for all other cases - // This handles: line_delta < 0, line_delta >= 3, or columns >= 128 - let end_col = col; // Use same column for end + // Handles: line_delta < 0, line_delta >= 3, multi-line spans, or columns >= 128 linetable.push( 0x80 | ((PyCodeLocationInfoKind::Long as u8) << 3) | ((entry_length - 1) as u8), ); write_signed_varint(&mut linetable, line_delta); - write_varint(&mut linetable, 0); // end_line delta = 0 - write_varint(&mut linetable, (col as u32) + 1); // column + 1 for encoding - write_varint(&mut linetable, (end_col as u32) + 1); // end_col + 1 + write_varint(&mut linetable, end_line_delta as u32); + write_varint(&mut linetable, (col as u32) + 1); + write_varint(&mut linetable, (end_col as u32) + 1); } prev_line = line; diff --git a/crates/compiler-core/src/bytecode.rs b/crates/compiler-core/src/bytecode.rs index 5569fa2012b..7675ef34863 100644 --- a/crates/compiler-core/src/bytecode.rs +++ b/crates/compiler-core/src/bytecode.rs @@ -258,7 +258,7 @@ impl ConstantBag for BasicBag { #[derive(Clone)] pub struct CodeObject { pub instructions: CodeUnits, - pub locations: Box<[SourceLocation]>, + pub locations: Box<[(SourceLocation, SourceLocation)]>, pub flags: CodeFlags, /// Number of positional-only arguments pub posonlyarg_count: u32, @@ -1483,14 +1483,14 @@ impl CodeObject { level: usize, ) -> fmt::Result { let label_targets = self.label_targets(); - let line_digits = (3).max(self.locations.last().unwrap().line.digits().get()); + let line_digits = (3).max(self.locations.last().unwrap().0.line.digits().get()); let offset_digits = (4).max(1 + self.instructions.len().ilog10() as usize); let mut last_line = OneIndexed::MAX; let mut arg_state = OpArgState::default(); for (offset, &instruction) in self.instructions.iter().enumerate() { let (instruction, arg) = arg_state.get(instruction); // optional line number - let line = self.locations[offset].line; + let line = self.locations[offset].0.line; if line != last_line { if last_line != OneIndexed::MAX { writeln!(f)?; diff --git a/crates/compiler-core/src/marshal.rs b/crates/compiler-core/src/marshal.rs index b30894ea065..5b528fe7e50 100644 --- a/crates/compiler-core/src/marshal.rs +++ b/crates/compiler-core/src/marshal.rs @@ -190,12 +190,17 @@ pub fn deserialize_code( let len = rdr.read_u32()?; let locations = (0..len) .map(|_| { - Ok(SourceLocation { + let start = SourceLocation { line: OneIndexed::new(rdr.read_u32()? as _).ok_or(MarshalError::InvalidLocation)?, character_offset: OneIndexed::from_zero_indexed(rdr.read_u32()? as _), - }) + }; + let end = SourceLocation { + line: OneIndexed::new(rdr.read_u32()? as _).ok_or(MarshalError::InvalidLocation)?, + character_offset: OneIndexed::from_zero_indexed(rdr.read_u32()? as _), + }; + Ok((start, end)) }) - .collect::>>()?; + .collect::>>()?; let flags = CodeFlags::from_bits_truncate(rdr.read_u16()?); @@ -648,9 +653,11 @@ pub fn serialize_code(buf: &mut W, code: &CodeObject) buf.write_slice(instructions_bytes); write_len(buf, code.locations.len()); - for loc in &*code.locations { - buf.write_u32(loc.line.get() as _); - buf.write_u32(loc.character_offset.to_zero_indexed() as _); + for (start, end) in &*code.locations { + buf.write_u32(start.line.get() as _); + buf.write_u32(start.character_offset.to_zero_indexed() as _); + buf.write_u32(end.line.get() as _); + buf.write_u32(end.character_offset.to_zero_indexed() as _); } buf.write_u16(code.flags.bits()); diff --git a/crates/vm/src/builtins/code.rs b/crates/vm/src/builtins/code.rs index b897ef9d311..32e53a5d376 100644 --- a/crates/vm/src/builtins/code.rs +++ b/crates/vm/src/builtins/code.rs @@ -467,20 +467,22 @@ impl Constructor for PyCode { .collect::>() .into_boxed_slice(); - // Create locations + // Create locations (start and end pairs) let row = if args.firstlineno > 0 { OneIndexed::new(args.firstlineno as usize).unwrap_or(OneIndexed::MIN) } else { OneIndexed::MIN }; - let locations: Box<[rustpython_compiler_core::SourceLocation]> = vec![ - rustpython_compiler_core::SourceLocation { - line: row, - character_offset: OneIndexed::from_zero_indexed(0), - }; - instructions.len() - ] - .into_boxed_slice(); + let loc = rustpython_compiler_core::SourceLocation { + line: row, + character_offset: OneIndexed::from_zero_indexed(0), + }; + let locations: Box< + [( + rustpython_compiler_core::SourceLocation, + rustpython_compiler_core::SourceLocation, + )], + > = vec![(loc, loc); instructions.len()].into_boxed_slice(); // Build the CodeObject let code = CodeObject { @@ -809,7 +811,6 @@ impl PyCode { Some(line + end_line_delta) }; - // Convert Option to PyObject (None or int) let line_obj = final_line.to_pyobject(vm); let end_line_obj = final_endline.to_pyobject(vm); let column_obj = column.to_pyobject(vm); diff --git a/crates/vm/src/builtins/frame.rs b/crates/vm/src/builtins/frame.rs index 3712b04e875..28c4e751476 100644 --- a/crates/vm/src/builtins/frame.rs +++ b/crates/vm/src/builtins/frame.rs @@ -58,12 +58,19 @@ impl Frame { #[pygetset] fn f_lasti(&self) -> u32 { - self.lasti() + // Return byte offset (each instruction is 2 bytes) for compatibility + self.lasti() * 2 } #[pygetset] pub fn f_lineno(&self) -> usize { - self.current_location().line.get() + // If lasti is 0, execution hasn't started yet - use first line number + // Similar to PyCode_Addr2Line which returns co_firstlineno for addr_q < 0 + if self.lasti() == 0 { + self.code.first_line_number.map(|n| n.get()).unwrap_or(1) + } else { + self.current_location().line.get() + } } #[pygetset] diff --git a/crates/vm/src/frame.rs b/crates/vm/src/frame.rs index 6acf0e84795..6500f578dca 100644 --- a/crates/vm/src/frame.rs +++ b/crates/vm/src/frame.rs @@ -172,7 +172,7 @@ impl Frame { } pub fn current_location(&self) -> SourceLocation { - self.code.locations[self.lasti() as usize - 1] + self.code.locations[self.lasti() as usize - 1].0 } pub fn lasti(&self) -> u32 { @@ -385,12 +385,12 @@ impl ExecutingFrame<'_> { // 2. Add new entry with current execution position (filename, lineno, code_object) to traceback. // 3. Unwind block stack till appropriate handler is found. - let loc = frame.code.locations[idx]; + let (loc, _end_loc) = frame.code.locations[idx]; let next = exception.__traceback__(); let new_traceback = PyTraceback::new( next, frame.object.to_owned(), - frame.lasti(), + frame.lasti() * 2, loc.line, ); vm_trace!("Adding to traceback: {:?} {:?}", new_traceback, loc.line); diff --git a/crates/vm/src/vm/mod.rs b/crates/vm/src/vm/mod.rs index 7c527b3e0da..ddbf7660f7d 100644 --- a/crates/vm/src/vm/mod.rs +++ b/crates/vm/src/vm/mod.rs @@ -508,6 +508,7 @@ impl VirtualMachine { pub fn compile_opts(&self) -> crate::compiler::CompileOpts { crate::compiler::CompileOpts { optimize: self.state.config.settings.optimize, + debug_ranges: self.state.config.settings.code_debug_ranges, } } diff --git a/crates/vm/src/vm/setting.rs b/crates/vm/src/vm/setting.rs index 53e2cef1160..8a307d1852b 100644 --- a/crates/vm/src/vm/setting.rs +++ b/crates/vm/src/vm/setting.rs @@ -57,7 +57,8 @@ pub struct Settings { // int tracemalloc; // int perf_profiling; // int import_time; - // int code_debug_ranges; + /// -X no_debug_ranges: disable column info in bytecode + pub code_debug_ranges: bool, // int show_ref_count; // int dump_refs; // wchar_t *dump_refs_file; @@ -192,6 +193,7 @@ impl Default for Settings { argv: vec![], hash_seed: None, faulthandler: false, + code_debug_ranges: true, buffered_stdio: true, check_hash_pycs_mode: CheckHashPycsMode::Default, allow_external_library: cfg!(feature = "importlib"), diff --git a/examples/dis.rs b/examples/dis.rs index 942643cd54b..1ca350603f9 100644 --- a/examples/dis.rs +++ b/examples/dis.rs @@ -53,7 +53,10 @@ fn main() -> Result<(), lexopt::Error> { return Err("expected at least one argument".into()); } - let opts = compiler::CompileOpts { optimize }; + let opts = compiler::CompileOpts { + optimize, + debug_ranges: true, + }; for script in &scripts { if script.exists() && script.is_file() { diff --git a/src/settings.rs b/src/settings.rs index a63f1a07ccc..54e66086932 100644 --- a/src/settings.rs +++ b/src/settings.rs @@ -270,6 +270,7 @@ pub fn parse_opts() -> Result<(Settings, RunMode), lexopt::Error> { "faulthandler" => settings.faulthandler = true, "warn_default_encoding" => settings.warn_default_encoding = true, "no_sig_int" => settings.install_signal_handlers = false, + "no_debug_ranges" => settings.code_debug_ranges = false, "int_max_str_digits" => { settings.int_max_str_digits = match value.unwrap().parse() { Ok(digits) if digits == 0 || digits >= 640 => digits, @@ -293,6 +294,9 @@ pub fn parse_opts() -> Result<(Settings, RunMode), lexopt::Error> { settings.warn_default_encoding = settings.warn_default_encoding || env_bool("PYTHONWARNDEFAULTENCODING"); settings.faulthandler = settings.faulthandler || env_bool("PYTHONFAULTHANDLER"); + if env_bool("PYTHONNODEBUGRANGES") { + settings.code_debug_ranges = false; + } if settings.dev_mode { settings.warnoptions.push("default".to_owned()); From 7b9f0d9657e86db7627c76e2414c105588838671 Mon Sep 17 00:00:00 2001 From: Ashwin Naren Date: Tue, 30 Dec 2025 00:07:03 -0800 Subject: [PATCH 0007/1575] More tkinter (#5784) * fix build * fix usage of Tcl_GetVar2Ex Signed-off-by: Ashwin Naren * no panic on mainloop Signed-off-by: Ashwin Naren * add static var Signed-off-by: Ashwin Naren * fix getvar Signed-off-by: Ashwin Naren * add globalgetvar support Signed-off-by: Ashwin Naren * formatting Signed-off-by: Ashwin Naren * address review * fix build * Auto-format: cargo fmt --all --------- Signed-off-by: Ashwin Naren Co-authored-by: Jeong YunWon Co-authored-by: github-actions[bot] --- .github/workflows/ci.yaml | 8 ++ crates/stdlib/Cargo.toml | 3 +- crates/stdlib/src/tkinter.rs | 144 ++++++++++++++++++++++++++--------- 3 files changed, 118 insertions(+), 37 deletions(-) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index dc5ce974e38..a873aa2d29d 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -157,6 +157,14 @@ jobs: cargo build --no-default-features --features ssl-openssl if: runner.os == 'Linux' + # - name: Install tk-dev for tkinter build + # run: sudo apt-get update && sudo apt-get install -y tk-dev + # if: runner.os == 'Linux' + + # - name: Test tkinter build + # run: cargo build --features tkinter + # if: runner.os == 'Linux' + - name: Test example projects run: cargo run --manifest-path example_projects/barebone/Cargo.toml diff --git a/crates/stdlib/Cargo.toml b/crates/stdlib/Cargo.toml index 4885319ea17..e943470a3af 100644 --- a/crates/stdlib/Cargo.toml +++ b/crates/stdlib/Cargo.toml @@ -21,7 +21,7 @@ ssl-rustls = ["ssl", "rustls", "rustls-native-certs", "rustls-pemfile", "rustls- ssl-rustls-fips = ["ssl-rustls", "aws-lc-rs/fips"] ssl-openssl = ["ssl", "openssl", "openssl-sys", "foreign-types-shared", "openssl-probe"] ssl-vendor = ["ssl-openssl", "openssl/vendored"] -tkinter = ["dep:tk-sys", "dep:tcl-sys"] +tkinter = ["dep:tk-sys", "dep:tcl-sys", "dep:widestring"] [dependencies] # rustpython crates @@ -90,6 +90,7 @@ bzip2 = "0.6" # tkinter tk-sys = { git = "https://github.com/arihant2math/tkinter.git", tag = "v0.2.0", optional = true } tcl-sys = { git = "https://github.com/arihant2math/tkinter.git", tag = "v0.2.0", optional = true } +widestring = { workspace = true, optional = true } chrono.workspace = true # uuid diff --git a/crates/stdlib/src/tkinter.rs b/crates/stdlib/src/tkinter.rs index 49dcdc5f84f..6b7c4387d6c 100644 --- a/crates/stdlib/src/tkinter.rs +++ b/crates/stdlib/src/tkinter.rs @@ -4,8 +4,9 @@ pub(crate) use self::_tkinter::make_module; #[pymodule] mod _tkinter { + use rustpython_vm::convert::IntoPyException; use rustpython_vm::types::Constructor; - use rustpython_vm::{PyObjectRef, PyPayload, PyRef, PyResult, VirtualMachine}; + use rustpython_vm::{Py, PyObjectRef, PyPayload, PyResult, VirtualMachine}; use rustpython_vm::builtins::{PyInt, PyStr, PyType}; use std::{ffi, ptr}; @@ -72,6 +73,7 @@ mod _tkinter { impl TclObject {} static QUIT_MAIN_LOOP: AtomicBool = AtomicBool::new(false); + static ERROR_IN_CMD: AtomicBool = AtomicBool::new(false); #[pyattr] #[pyclass(name = "tkapp")] @@ -125,7 +127,7 @@ mod _tkinter { interactive: i32, #[pyarg(any)] wantobjects: i32, - #[pyarg(any, default = "true")] + #[pyarg(any, default = true)] want_tk: bool, #[pyarg(any)] sync: i32, @@ -136,11 +138,7 @@ mod _tkinter { impl Constructor for TkApp { type Args = TkAppConstructorArgs; - fn py_new( - _zelf: PyRef, - args: Self::Args, - vm: &VirtualMachine, - ) -> PyResult { + fn py_new(_cls: &Py, args: Self::Args, vm: &VirtualMachine) -> PyResult { create(args, vm) } } @@ -168,10 +166,18 @@ mod _tkinter { ))) } + #[derive(Debug, FromArgs)] + struct TkAppGetVarArgs { + #[pyarg(any)] + name: PyObjectRef, + #[pyarg(any, default)] + name2: Option, + } + // TODO: DISALLOW_INSTANTIATION #[pyclass(with(Constructor))] impl TkApp { - fn from_bool(&self, obj: *mut tk_sys::Tcl_Obj) -> bool { + fn tcl_obj_to_bool(&self, obj: *mut tk_sys::Tcl_Obj) -> bool { let mut res = -1; unsafe { if tk_sys::Tcl_GetBooleanFromObj(self.interpreter, obj, &mut res) @@ -184,16 +190,16 @@ mod _tkinter { res != 0 } - fn from_object( + fn tcl_obj_to_pyobject( &self, obj: *mut tk_sys::Tcl_Obj, vm: &VirtualMachine, ) -> PyResult { let type_ptr = unsafe { (*obj).typePtr }; - if type_ptr == ptr::null() { + if type_ptr.is_null() { return self.unicode_from_object(obj, vm); } else if type_ptr == self.old_boolean_type || type_ptr == self.boolean_type { - return Ok(vm.ctx.new_bool(self.from_bool(obj)).into()); + return Ok(vm.ctx.new_bool(self.tcl_obj_to_bool(obj)).into()); } else if type_ptr == self.string_type || type_ptr == self.utf32_string_type || type_ptr == self.pixel_type @@ -202,7 +208,7 @@ mod _tkinter { } // TODO: handle other types - return Ok(TclObject { value: obj }.into_pyobject(vm)); + Ok(TclObject { value: obj }.into_pyobject(vm)) } fn unicode_from_string( @@ -226,8 +232,8 @@ mod _tkinter { vm: &VirtualMachine, ) -> PyResult { let type_ptr = unsafe { (*obj).typePtr }; - if type_ptr != ptr::null() - && self.interpreter != ptr::null_mut() + if !type_ptr.is_null() + && !self.interpreter.is_null() && (type_ptr == self.string_type || type_ptr == self.utf32_string_type) { let len = ptr::null_mut(); @@ -247,30 +253,75 @@ mod _tkinter { Self::unicode_from_string(s, len as _, vm) } - #[pymethod] - fn getvar(&self, arg: PyObjectRef, vm: &VirtualMachine) -> PyResult { + fn var_invoke(&self) { + if self.threaded && self.thread_id != Some(unsafe { tk_sys::Tcl_GetCurrentThread() }) { + // TODO: do stuff + } + } + + fn inner_getvar( + &self, + args: TkAppGetVarArgs, + flags: u32, + vm: &VirtualMachine, + ) -> PyResult { + let TkAppGetVarArgs { name, name2 } = args; // TODO: technically not thread safe - let name = varname_converter(arg, vm)?; + let name = varname_converter(name, vm)?; + let name = ffi::CString::new(name).map_err(|e| e.into_pyexception(vm))?; + let name2 = + ffi::CString::new(name2.unwrap_or_default()).map_err(|e| e.into_pyexception(vm))?; + let name2_ptr = if name2.is_empty() { + ptr::null() + } else { + name2.as_ptr() + }; let res = unsafe { tk_sys::Tcl_GetVar2Ex( self.interpreter, - ptr::null(), name.as_ptr() as _, - tk_sys::TCL_LEAVE_ERR_MSG as _, + name2_ptr as _, + flags as _, ) }; - if res == ptr::null_mut() { - todo!(); + if res.is_null() { + // TODO: Should be tk error + unsafe { + let err_obj = tk_sys::Tcl_GetObjResult(self.interpreter); + let err_str_obj = tk_sys::Tcl_GetString(err_obj); + let err_cstr = ffi::CStr::from_ptr(err_str_obj as _); + return Err(vm.new_type_error(format!("{err_cstr:?}"))); + } } let res = if self.want_objects { - self.from_object(res, vm) + self.tcl_obj_to_pyobject(res, vm) } else { self.unicode_from_object(res, vm) }?; Ok(res) } + #[pymethod] + fn getvar(&self, args: TkAppGetVarArgs, vm: &VirtualMachine) -> PyResult { + self.var_invoke(); + self.inner_getvar(args, tk_sys::TCL_LEAVE_ERR_MSG, vm) + } + + #[pymethod] + fn globalgetvar( + &self, + args: TkAppGetVarArgs, + vm: &VirtualMachine, + ) -> PyResult { + self.var_invoke(); + self.inner_getvar( + args, + tk_sys::TCL_LEAVE_ERR_MSG | tk_sys::TCL_GLOBAL_ONLY, + vm, + ) + } + #[pymethod] fn getint(&self, arg: PyObjectRef, vm: &VirtualMachine) -> PyResult { if let Some(int) = arg.downcast_ref::() { @@ -289,7 +340,21 @@ mod _tkinter { #[pymethod] fn mainloop(&self, threshold: Option) -> PyResult<()> { let threshold = threshold.unwrap_or(0); - todo!(); + // self.dispatching = true; + QUIT_MAIN_LOOP.store(false, Ordering::Relaxed); + while unsafe { tk_sys::Tk_GetNumMainWindows() } > threshold + && !QUIT_MAIN_LOOP.load(Ordering::Relaxed) + && !ERROR_IN_CMD.load(Ordering::Relaxed) + { + if self.threaded { + unsafe { tk_sys::Tcl_DoOneEvent(0 as _) }; + } else { + unsafe { tk_sys::Tcl_DoOneEvent(tk_sys::TCL_DONT_WAIT as _) }; + // TODO: sleep for the proper time + std::thread::sleep(std::time::Duration::from_millis(1)); + } + } + Ok(()) } #[pymethod] @@ -299,13 +364,15 @@ mod _tkinter { } #[pyfunction] - fn create(args: TkAppConstructorArgs, vm: &VirtualMachine) -> PyResult { + fn create(args: TkAppConstructorArgs, vm: &VirtualMachine) -> PyResult { unsafe { let interp = tk_sys::Tcl_CreateInterp(); let want_objects = args.wantobjects != 0; - let threaded = { + let threaded = !{ let part1 = String::from("tcl_platform"); let part2 = String::from("threaded"); + let part1 = ffi::CString::new(part1).map_err(|e| e.into_pyexception(vm))?; + let part2 = ffi::CString::new(part2).map_err(|e| e.into_pyexception(vm))?; let part1_ptr = part1.as_ptr(); let part2_ptr = part2.as_ptr(); tk_sys::Tcl_GetVar2Ex( @@ -314,7 +381,8 @@ mod _tkinter { part2_ptr as _, tk_sys::TCL_GLOBAL_ONLY as ffi::c_int, ) - } != ptr::null_mut(); + } + .is_null(); let thread_id = tk_sys::Tcl_GetCurrentThread(); let dispatching = false; let trace = None; @@ -339,8 +407,8 @@ mod _tkinter { let double_type = tk_sys::Tcl_GetObjType(double_str.as_ptr() as _); let int_str = String::from("int"); let int_type = tk_sys::Tcl_GetObjType(int_str.as_ptr() as _); - let int_type = if int_type == ptr::null() { - let mut value = *tk_sys::Tcl_NewIntObj(0); + let int_type = if int_type.is_null() { + let mut value = *tk_sys::Tcl_NewWideIntObj(0); let res = value.typePtr; tk_sys::Tcl_DecrRefCount(&mut value); res @@ -374,33 +442,37 @@ mod _tkinter { } if args.interactive != 0 { - tk_sys::Tcl_SetVar( + tk_sys::Tcl_SetVar2( interp, "tcl_interactive".as_ptr() as _, + ptr::null(), "1".as_ptr() as _, tk_sys::TCL_GLOBAL_ONLY as i32, ); } else { - tk_sys::Tcl_SetVar( + tk_sys::Tcl_SetVar2( interp, "tcl_interactive".as_ptr() as _, + ptr::null(), "0".as_ptr() as _, tk_sys::TCL_GLOBAL_ONLY as i32, ); } let argv0 = args.class_name.clone().to_lowercase(); - tk_sys::Tcl_SetVar( + tk_sys::Tcl_SetVar2( interp, "argv0".as_ptr() as _, + ptr::null(), argv0.as_ptr() as _, tk_sys::TCL_GLOBAL_ONLY as i32, ); if !args.want_tk { - tk_sys::Tcl_SetVar( + tk_sys::Tcl_SetVar2( interp, "_tkinter_skip_tk_init".as_ptr() as _, + ptr::null(), "1".as_ptr() as _, tk_sys::TCL_GLOBAL_ONLY as i32, ); @@ -418,11 +490,12 @@ mod _tkinter { argv.push_str("-use "); argv.push_str(&args.use_.unwrap()); } - argv.push_str("\0"); + argv.push('\0'); let argv_ptr = argv.as_ptr() as *mut *mut i8; - tk_sys::Tcl_SetVar( + tk_sys::Tcl_SetVar2( interp, "argv".as_ptr() as _, + ptr::null(), argv_ptr as *const i8, tk_sys::TCL_GLOBAL_ONLY as i32, ); @@ -460,8 +533,7 @@ mod _tkinter { string_type, utf32_string_type, pixel_type, - } - .into_pyobject(vm)) + }) } } From 9fcc471c94844cbfeaa4b6c1853217ea18330f53 Mon Sep 17 00:00:00 2001 From: "Jeong, YunWon" <69878+youknowone@users.noreply.github.com> Date: Tue, 30 Dec 2025 17:36:15 +0900 Subject: [PATCH 0008/1575] Prevent tb_next loop (#6596) --- Lib/test/test_raise.py | 1 - crates/vm/src/builtins/traceback.rs | 26 ++++++++++++++++++++++---- 2 files changed, 22 insertions(+), 5 deletions(-) diff --git a/Lib/test/test_raise.py b/Lib/test/test_raise.py index 53fce0a501d..605169f4b40 100644 --- a/Lib/test/test_raise.py +++ b/Lib/test/test_raise.py @@ -244,7 +244,6 @@ class TestTracebackType(unittest.TestCase): def raiser(self): raise ValueError - @unittest.expectedFailure # TODO: RUSTPYTHON def test_attrs(self): try: self.raiser() diff --git a/crates/vm/src/builtins/traceback.rs b/crates/vm/src/builtins/traceback.rs index 84493e7125f..675025cd6b6 100644 --- a/crates/vm/src/builtins/traceback.rs +++ b/crates/vm/src/builtins/traceback.rs @@ -1,7 +1,7 @@ use super::PyType; use crate::{ - Context, Py, PyPayload, PyRef, PyResult, VirtualMachine, class::PyClassImpl, frame::FrameRef, - types::Constructor, + AsObject, Context, Py, PyPayload, PyRef, PyResult, VirtualMachine, class::PyClassImpl, + frame::FrameRef, types::Constructor, }; use rustpython_common::lock::PyMutex; use rustpython_compiler_core::OneIndexed; @@ -63,8 +63,26 @@ impl PyTraceback { } #[pygetset(setter)] - fn set_tb_next(&self, value: Option>) { - *self.next.lock() = value; + fn set_tb_next( + zelf: &Py, + value: Option>, + vm: &VirtualMachine, + ) -> PyResult<()> { + if let Some(ref new_next) = value { + let mut cursor = new_next.clone(); + loop { + if cursor.is(zelf) { + return Err(vm.new_value_error("traceback loop detected".to_owned())); + } + let next = cursor.next.lock().clone(); + match next { + Some(n) => cursor = n, + None => break, + } + } + } + *zelf.next.lock() = value; + Ok(()) } } From b98f06214d0f6e8b2dc642e354e53c7ea566d4d4 Mon Sep 17 00:00:00 2001 From: "Jeong, YunWon" <69878+youknowone@users.noreply.github.com> Date: Wed, 31 Dec 2025 01:42:11 +0900 Subject: [PATCH 0009/1575] Update issubclass and make mro representation fit to CPython (#5866) --- crates/vm/src/builtins/type.rs | 56 ++++++++++++++------------------ crates/vm/src/object/core.rs | 8 ++++- crates/vm/src/types/slot.rs | 7 ++-- crates/vm/src/types/slot_defs.rs | 5 +-- 4 files changed, 39 insertions(+), 37 deletions(-) diff --git a/crates/vm/src/builtins/type.rs b/crates/vm/src/builtins/type.rs index 3eca5edc478..178131c96d6 100644 --- a/crates/vm/src/builtins/type.rs +++ b/crates/vm/src/builtins/type.rs @@ -50,7 +50,8 @@ unsafe impl crate::object::Traverse for PyType { fn traverse(&self, tracer_fn: &mut crate::object::TraverseFn<'_>) { self.base.traverse(tracer_fn); self.bases.traverse(tracer_fn); - self.mro.traverse(tracer_fn); + // mro contains self as mro[0], so skip traversing to avoid circular reference + // self.mro.traverse(tracer_fn); self.subclasses.traverse(tracer_fn); self.attributes .read_recursive() @@ -341,6 +342,7 @@ impl PyType { metaclass, None, ); + new_type.mro.write().insert(0, new_type.clone()); new_type.init_slots(ctx); @@ -393,6 +395,7 @@ impl PyType { metaclass, None, ); + new_type.mro.write().insert(0, new_type.clone()); // Note: inherit_slots is called in PyClassImpl::init_class after // slots are fully initialized by make_slots() @@ -413,9 +416,8 @@ impl PyType { } pub(crate) fn init_slots(&self, ctx: &Context) { - // Inherit slots from MRO - // Note: self.mro does NOT include self, so we iterate all elements - let mro: Vec<_> = self.mro.read().iter().cloned().collect(); + // Inherit slots from MRO (mro[0] is self, so skip it) + let mro: Vec<_> = self.mro.read()[1..].to_vec(); for base in mro.iter() { self.inherit_slots(base); } @@ -424,7 +426,8 @@ impl PyType { #[allow(clippy::mutable_key_type)] let mut slot_name_set = std::collections::HashSet::new(); - for cls in self.mro.read().iter() { + // mro[0] is self, so skip it; self.attributes is checked separately below + for cls in self.mro.read()[1..].iter() { for &name in cls.attributes.read().keys() { if name.as_bytes().starts_with(b"__") && name.as_bytes().ends_with(b"__") { slot_name_set.insert(name); @@ -503,18 +506,12 @@ impl PyType { /// Equivalent to CPython's find_name_in_mro /// Look in tp_dict of types in MRO - bypasses descriptors and other attribute access machinery fn find_name_in_mro(&self, name: &'static PyStrInterned) -> Option { - // First check in our own dict - if let Some(value) = self.attributes.read().get(name) { - return Some(value.clone()); - } - - // Then check in MRO - for base in self.mro.read().iter() { - if let Some(value) = base.attributes.read().get(name) { + // mro[0] is self, so we just iterate through the entire MRO + for cls in self.mro.read().iter() { + if let Some(value) = cls.attributes.read().get(name) { return Some(value.clone()); } } - None } @@ -530,8 +527,7 @@ impl PyType { } pub fn get_super_attr(&self, attr_name: &'static PyStrInterned) -> Option { - self.mro - .read() + self.mro.read()[1..] .iter() .find_map(|class| class.attributes.read().get(attr_name).cloned()) } @@ -539,9 +535,7 @@ impl PyType { // This is the internal has_attr implementation for fast lookup on a class. pub fn has_attr(&self, attr_name: &'static PyStrInterned) -> bool { self.attributes.read().contains_key(attr_name) - || self - .mro - .read() + || self.mro.read()[1..] .iter() .any(|c| c.attributes.read().contains_key(attr_name)) } @@ -550,10 +544,8 @@ impl PyType { // Gather all members here: let mut attributes = PyAttributes::default(); - for bc in core::iter::once(self) - .chain(self.mro.read().iter().map(|cls| -> &Self { cls })) - .rev() - { + // mro[0] is self, so we iterate through the entire MRO in reverse + for bc in self.mro.read().iter().map(|cls| -> &Self { cls }).rev() { for (name, value) in bc.attributes.read().iter() { attributes.insert(name.to_owned(), value.clone()); } @@ -661,22 +653,21 @@ impl Py { /// so only use this if `cls` is known to have not overridden the base __subclasscheck__ magic /// method. pub fn fast_issubclass(&self, cls: &impl Borrow) -> bool { - self.as_object().is(cls.borrow()) || self.mro.read().iter().any(|c| c.is(cls.borrow())) + self.as_object().is(cls.borrow()) || self.mro.read()[1..].iter().any(|c| c.is(cls.borrow())) } pub fn mro_map_collect(&self, f: F) -> Vec where F: Fn(&Self) -> R, { - core::iter::once(self) - .chain(self.mro.read().iter().map(|x| x.deref())) - .map(f) - .collect() + self.mro.read().iter().map(|x| x.deref()).map(f).collect() } pub fn mro_collect(&self) -> Vec> { - core::iter::once(self) - .chain(self.mro.read().iter().map(|x| x.deref())) + self.mro + .read() + .iter() + .map(|x| x.deref()) .map(|x| x.to_owned()) .collect() } @@ -745,8 +736,11 @@ impl PyType { *zelf.bases.write() = bases; // Recursively update the mros of this class and all subclasses fn update_mro_recursively(cls: &PyType, vm: &VirtualMachine) -> PyResult<()> { - *cls.mro.write() = + let mut mro = PyType::resolve_mro(&cls.bases.read()).map_err(|msg| vm.new_type_error(msg))?; + // Preserve self (mro[0]) when updating MRO + mro.insert(0, cls.mro.read()[0].to_owned()); + *cls.mro.write() = mro; for subclass in cls.subclasses.write().iter() { let subclass = subclass.upgrade().unwrap(); let subclass: &Py = subclass.downcast_ref().unwrap(); diff --git a/crates/vm/src/object/core.rs b/crates/vm/src/object/core.rs index cee3fac266e..e904117b06a 100644 --- a/crates/vm/src/object/core.rs +++ b/crates/vm/src/object/core.rs @@ -1310,12 +1310,16 @@ pub(crate) fn init_type_hierarchy() -> (PyTypeRef, PyTypeRef, PyTypeRef) { ptr::write(&mut (*type_type_ptr).typ, PyAtomicRef::from(type_type)); let object_type = PyTypeRef::from_raw(object_type_ptr.cast()); + // object's mro is [object] + (*object_type_ptr).payload.mro = PyRwLock::new(vec![object_type.clone()]); - (*type_type_ptr).payload.mro = PyRwLock::new(vec![object_type.clone()]); (*type_type_ptr).payload.bases = PyRwLock::new(vec![object_type.clone()]); (*type_type_ptr).payload.base = Some(object_type.clone()); let type_type = PyTypeRef::from_raw(type_type_ptr.cast()); + // type's mro is [type, object] + (*type_type_ptr).payload.mro = + PyRwLock::new(vec![type_type.clone(), object_type.clone()]); (type_type, object_type) } @@ -1331,6 +1335,8 @@ pub(crate) fn init_type_hierarchy() -> (PyTypeRef, PyTypeRef, PyTypeRef) { heaptype_ext: None, }; let weakref_type = PyRef::new_ref(weakref_type, type_type.clone(), None); + // weakref's mro is [weakref, object] + weakref_type.mro.write().insert(0, weakref_type.clone()); object_type.subclasses.write().push( type_type diff --git a/crates/vm/src/types/slot.rs b/crates/vm/src/types/slot.rs index 9d7d09f2d18..085cfea5cee 100644 --- a/crates/vm/src/types/slot.rs +++ b/crates/vm/src/types/slot.rs @@ -760,8 +760,9 @@ impl PyType { let in_self = cmp_names.iter().any(|n| attrs.contains_key(*n)); drop(attrs); + // mro[0] is self, so skip it since we already checked self above in_self - || self.mro.read().iter().any(|cls| { + || self.mro.read()[1..].iter().any(|cls| { let attrs = cls.attributes.read(); cmp_names.iter().any(|n| { if let Some(attr) = attrs.get(*n) { @@ -1273,8 +1274,8 @@ impl PyType { return None; } - // Look up in MRO - for cls in self.mro.read().iter() { + // Look up in MRO (mro[0] is self, so skip it) + for cls in self.mro.read()[1..].iter() { if let Some(attr) = cls.attributes.read().get(name).cloned() { if let Some(func) = try_extract(&attr) { return Some(func); diff --git a/crates/vm/src/types/slot_defs.rs b/crates/vm/src/types/slot_defs.rs index 0d8dbf4b0cc..958adb2f27e 100644 --- a/crates/vm/src/types/slot_defs.rs +++ b/crates/vm/src/types/slot_defs.rs @@ -405,8 +405,9 @@ impl SlotAccessor { /// Inherit slot value from MRO pub fn inherit_from_mro(&self, typ: &crate::builtins::PyType) { - // Note: typ.mro does NOT include typ itself - let mro = typ.mro.read(); + // mro[0] is self, so skip it + let mro_guard = typ.mro.read(); + let mro = &mro_guard[1..]; macro_rules! inherit_main { ($slot:ident) => {{ From ca1c4c1f715c32ab6a60f6d395c9cbc6df3c56e4 Mon Sep 17 00:00:00 2001 From: Vinh Nguyen <89290241+ntvinh2005@users.noreply.github.com> Date: Tue, 30 Dec 2025 11:54:50 -0500 Subject: [PATCH 0010/1575] feat: Sync code.py + test_code_module.py from CPython v3.13.1 (#6181) * feat: Sync code module from cpython 3.13 * also udpate test_code_module --------- Co-authored-by: Jeong YunWon --- Lib/code.py | 203 +++++++++++++++++++++++----------- Lib/test/test_code_module.py | 208 ++++++++++++++++++++++++++++++++--- 2 files changed, 326 insertions(+), 85 deletions(-) diff --git a/Lib/code.py b/Lib/code.py index 2bd5fa3e795..2777c311187 100644 --- a/Lib/code.py +++ b/Lib/code.py @@ -5,6 +5,7 @@ # Inspired by similar code by Jeff Epler and Fredrik Lundh. +import builtins import sys import traceback from codeop import CommandCompiler, compile_command @@ -12,6 +13,7 @@ __all__ = ["InteractiveInterpreter", "InteractiveConsole", "interact", "compile_command"] + class InteractiveInterpreter: """Base class for InteractiveConsole. @@ -24,10 +26,10 @@ class InteractiveInterpreter: def __init__(self, locals=None): """Constructor. - The optional 'locals' argument specifies the dictionary in - which code will be executed; it defaults to a newly created - dictionary with key "__name__" set to "__console__" and key - "__doc__" set to None. + The optional 'locals' argument specifies a mapping to use as the + namespace in which code will be executed; it defaults to a newly + created dictionary with key "__name__" set to "__console__" and + key "__doc__" set to None. """ if locals is None: @@ -63,7 +65,7 @@ def runsource(self, source, filename="", symbol="single"): code = self.compile(source, filename, symbol) except (OverflowError, SyntaxError, ValueError): # Case 1 - self.showsyntaxerror(filename) + self.showsyntaxerror(filename, source=source) return False if code is None: @@ -93,7 +95,7 @@ def runcode(self, code): except: self.showtraceback() - def showsyntaxerror(self, filename=None): + def showsyntaxerror(self, filename=None, **kwargs): """Display the syntax error that just occurred. This doesn't display a stack trace because there isn't one. @@ -105,29 +107,14 @@ def showsyntaxerror(self, filename=None): The output is written by self.write(), below. """ - type, value, tb = sys.exc_info() - sys.last_exc = value - sys.last_type = type - sys.last_value = value - sys.last_traceback = tb - if filename and type is SyntaxError: - # Work hard to stuff the correct filename in the exception - try: - msg, (dummy_filename, lineno, offset, line) = value.args - except ValueError: - # Not the format we expect; leave it alone - pass - else: - # Stuff in the right filename - value = SyntaxError(msg, (filename, lineno, offset, line)) - sys.last_exc = sys.last_value = value - if sys.excepthook is sys.__excepthook__: - lines = traceback.format_exception_only(type, value) - self.write(''.join(lines)) - else: - # If someone has set sys.excepthook, we let that take precedence - # over self.write - sys.excepthook(type, value, tb) + try: + typ, value, tb = sys.exc_info() + if filename and issubclass(typ, SyntaxError): + value.filename = filename + source = kwargs.pop('source', "") + self._showtraceback(typ, value, None, source) + finally: + typ = value = tb = None def showtraceback(self): """Display the exception that just occurred. @@ -137,19 +124,46 @@ def showtraceback(self): The output is written by self.write(), below. """ - sys.last_type, sys.last_value, last_tb = ei = sys.exc_info() - sys.last_traceback = last_tb - sys.last_exc = ei[1] try: - lines = traceback.format_exception(ei[0], ei[1], last_tb.tb_next) - if sys.excepthook is sys.__excepthook__: - self.write(''.join(lines)) - else: - # If someone has set sys.excepthook, we let that take precedence - # over self.write - sys.excepthook(ei[0], ei[1], last_tb) + typ, value, tb = sys.exc_info() + self._showtraceback(typ, value, tb.tb_next, '') finally: - last_tb = ei = None + typ = value = tb = None + + def _showtraceback(self, typ, value, tb, source): + sys.last_type = typ + sys.last_traceback = tb + value = value.with_traceback(tb) + # Set the line of text that the exception refers to + lines = source.splitlines() + if (source and typ is SyntaxError + and not value.text and value.lineno is not None + and len(lines) >= value.lineno): + value.text = lines[value.lineno - 1] + sys.last_exc = sys.last_value = value = value.with_traceback(tb) + if sys.excepthook is sys.__excepthook__: + self._excepthook(typ, value, tb) + else: + # If someone has set sys.excepthook, we let that take precedence + # over self.write + try: + sys.excepthook(typ, value, tb) + except SystemExit: + raise + except BaseException as e: + e.__context__ = None + e = e.with_traceback(e.__traceback__.tb_next) + print('Error in sys.excepthook:', file=sys.stderr) + sys.__excepthook__(type(e), e, e.__traceback__) + print(file=sys.stderr) + print('Original exception was:', file=sys.stderr) + sys.__excepthook__(typ, value, tb) + + def _excepthook(self, typ, value, tb): + # This method is being overwritten in + # _pyrepl.console.InteractiveColoredConsole + lines = traceback.format_exception(typ, value, tb) + self.write(''.join(lines)) def write(self, data): """Write a string. @@ -169,7 +183,7 @@ class InteractiveConsole(InteractiveInterpreter): """ - def __init__(self, locals=None, filename=""): + def __init__(self, locals=None, filename="", *, local_exit=False): """Constructor. The optional locals argument will be passed to the @@ -181,6 +195,7 @@ def __init__(self, locals=None, filename=""): """ InteractiveInterpreter.__init__(self, locals) self.filename = filename + self.local_exit = local_exit self.resetbuffer() def resetbuffer(self): @@ -219,29 +234,66 @@ def interact(self, banner=None, exitmsg=None): elif banner: self.write("%s\n" % str(banner)) more = 0 - while 1: - try: - if more: - prompt = sys.ps2 - else: - prompt = sys.ps1 + + # When the user uses exit() or quit() in their interactive shell + # they probably just want to exit the created shell, not the whole + # process. exit and quit in builtins closes sys.stdin which makes + # it super difficult to restore + # + # When self.local_exit is True, we overwrite the builtins so + # exit() and quit() only raises SystemExit and we can catch that + # to only exit the interactive shell + + _exit = None + _quit = None + + if self.local_exit: + if hasattr(builtins, "exit"): + _exit = builtins.exit + builtins.exit = Quitter("exit") + + if hasattr(builtins, "quit"): + _quit = builtins.quit + builtins.quit = Quitter("quit") + + try: + while True: try: - line = self.raw_input(prompt) - except EOFError: - self.write("\n") - break - else: - more = self.push(line) - except KeyboardInterrupt: - self.write("\nKeyboardInterrupt\n") - self.resetbuffer() - more = 0 - if exitmsg is None: - self.write('now exiting %s...\n' % self.__class__.__name__) - elif exitmsg != '': - self.write('%s\n' % exitmsg) - - def push(self, line): + if more: + prompt = sys.ps2 + else: + prompt = sys.ps1 + try: + line = self.raw_input(prompt) + except EOFError: + self.write("\n") + break + else: + more = self.push(line) + except KeyboardInterrupt: + self.write("\nKeyboardInterrupt\n") + self.resetbuffer() + more = 0 + except SystemExit as e: + if self.local_exit: + self.write("\n") + break + else: + raise e + finally: + # restore exit and quit in builtins if they were modified + if _exit is not None: + builtins.exit = _exit + + if _quit is not None: + builtins.quit = _quit + + if exitmsg is None: + self.write('now exiting %s...\n' % self.__class__.__name__) + elif exitmsg != '': + self.write('%s\n' % exitmsg) + + def push(self, line, filename=None, _symbol="single"): """Push a line to the interpreter. The line should not have a trailing newline; it may have @@ -257,7 +309,9 @@ def push(self, line): """ self.buffer.append(line) source = "\n".join(self.buffer) - more = self.runsource(source, self.filename) + if filename is None: + filename = self.filename + more = self.runsource(source, filename, symbol=_symbol) if not more: self.resetbuffer() return more @@ -276,8 +330,22 @@ def raw_input(self, prompt=""): return input(prompt) +class Quitter: + def __init__(self, name): + self.name = name + if sys.platform == "win32": + self.eof = 'Ctrl-Z plus Return' + else: + self.eof = 'Ctrl-D (i.e. EOF)' + + def __repr__(self): + return f'Use {self.name} or {self.eof} to exit' + + def __call__(self, code=None): + raise SystemExit(code) + -def interact(banner=None, readfunc=None, local=None, exitmsg=None): +def interact(banner=None, readfunc=None, local=None, exitmsg=None, local_exit=False): """Closely emulate the interactive Python interpreter. This is a backwards compatible interface to the InteractiveConsole @@ -290,9 +358,10 @@ def interact(banner=None, readfunc=None, local=None, exitmsg=None): readfunc -- if not None, replaces InteractiveConsole.raw_input() local -- passed to InteractiveInterpreter.__init__() exitmsg -- passed to InteractiveConsole.interact() + local_exit -- passed to InteractiveConsole.__init__() """ - console = InteractiveConsole(local) + console = InteractiveConsole(local, local_exit=local_exit) if readfunc is not None: console.raw_input = readfunc else: @@ -308,7 +377,7 @@ def interact(banner=None, readfunc=None, local=None, exitmsg=None): parser = argparse.ArgumentParser() parser.add_argument('-q', action='store_true', - help="don't print version and copyright messages") + help="don't print version and copyright messages") args = parser.parse_args() if args.q or sys.flags.quiet: banner = '' diff --git a/Lib/test/test_code_module.py b/Lib/test/test_code_module.py index 5ac17ef16ea..346cf6746a7 100644 --- a/Lib/test/test_code_module.py +++ b/Lib/test/test_code_module.py @@ -1,20 +1,17 @@ "Test InteractiveConsole and InteractiveInterpreter from code module" import sys +import traceback import unittest from textwrap import dedent from contextlib import ExitStack from unittest import mock +from test.support import force_not_colorized_test_class from test.support import import_helper - code = import_helper.import_module('code') -class TestInteractiveConsole(unittest.TestCase): - - def setUp(self): - self.console = code.InteractiveConsole() - self.mock_sys() +class MockSys: def mock_sys(self): "Mock system environment for InteractiveConsole" @@ -32,6 +29,15 @@ def mock_sys(self): del self.sysmod.ps1 del self.sysmod.ps2 + +@force_not_colorized_test_class +class TestInteractiveConsole(unittest.TestCase, MockSys): + maxDiff = None + + def setUp(self): + self.console = code.InteractiveConsole() + self.mock_sys() + def test_ps1(self): self.infunc.side_effect = EOFError('Finished') self.console.interact() @@ -44,9 +50,9 @@ def test_ps2(self): self.infunc.side_effect = EOFError('Finished') self.console.interact() self.assertEqual(self.sysmod.ps2, '... ') - self.sysmod.ps1 = 'custom2> ' + self.sysmod.ps2 = 'custom2> ' self.console.interact() - self.assertEqual(self.sysmod.ps1, 'custom2> ') + self.assertEqual(self.sysmod.ps2, 'custom2> ') def test_console_stderr(self): self.infunc.side_effect = ["'antioch'", "", EOFError('Finished')] @@ -57,22 +63,162 @@ def test_console_stderr(self): else: raise AssertionError("no console stdout") + # TODO: RUSTPYTHON + @unittest.expectedFailure def test_syntax_error(self): - self.infunc.side_effect = ["undefined", EOFError('Finished')] + self.infunc.side_effect = ["def f():", + " x = ?", + "", + EOFError('Finished')] self.console.interact() - for call in self.stderr.method_calls: - if 'NameError' in ''.join(call[1]): - break - else: - raise AssertionError("No syntax error from console") + output = ''.join(''.join(call[1]) for call in self.stderr.method_calls) + output = output[output.index('(InteractiveConsole)'):] + output = output[:output.index('\nnow exiting')] + self.assertEqual(output.splitlines()[1:], [ + ' File "", line 2', + ' x = ?', + ' ^', + 'SyntaxError: invalid syntax']) + self.assertIs(self.sysmod.last_type, SyntaxError) + self.assertIs(type(self.sysmod.last_value), SyntaxError) + self.assertIsNone(self.sysmod.last_traceback) + self.assertIsNone(self.sysmod.last_value.__traceback__) + self.assertIs(self.sysmod.last_exc, self.sysmod.last_value) + + # TODO: RUSTPYTHON + @unittest.expectedFailure + def test_indentation_error(self): + self.infunc.side_effect = [" 1", EOFError('Finished')] + self.console.interact() + output = ''.join(''.join(call[1]) for call in self.stderr.method_calls) + output = output[output.index('(InteractiveConsole)'):] + output = output[:output.index('\nnow exiting')] + self.assertEqual(output.splitlines()[1:], [ + ' File "", line 1', + ' 1', + 'IndentationError: unexpected indent']) + self.assertIs(self.sysmod.last_type, IndentationError) + self.assertIs(type(self.sysmod.last_value), IndentationError) + self.assertIsNone(self.sysmod.last_traceback) + self.assertIsNone(self.sysmod.last_value.__traceback__) + self.assertIs(self.sysmod.last_exc, self.sysmod.last_value) + + # TODO: RUSTPYTHON + @unittest.expectedFailure + def test_unicode_error(self): + self.infunc.side_effect = ["'\ud800'", EOFError('Finished')] + self.console.interact() + output = ''.join(''.join(call[1]) for call in self.stderr.method_calls) + output = output[output.index('(InteractiveConsole)'):] + output = output[output.index('\n') + 1:] + self.assertTrue(output.startswith('UnicodeEncodeError: '), output) + self.assertIs(self.sysmod.last_type, UnicodeEncodeError) + self.assertIs(type(self.sysmod.last_value), UnicodeEncodeError) + self.assertIsNone(self.sysmod.last_traceback) + self.assertIsNone(self.sysmod.last_value.__traceback__) + self.assertIs(self.sysmod.last_exc, self.sysmod.last_value) def test_sysexcepthook(self): - self.infunc.side_effect = ["raise ValueError('')", + self.infunc.side_effect = ["def f():", + " raise ValueError('BOOM!')", + "", + "f()", + EOFError('Finished')] + hook = mock.Mock() + self.sysmod.excepthook = hook + self.console.interact() + hook.assert_called() + hook.assert_called_with(self.sysmod.last_type, + self.sysmod.last_value, + self.sysmod.last_traceback) + self.assertIs(self.sysmod.last_type, ValueError) + self.assertIs(type(self.sysmod.last_value), ValueError) + self.assertIs(self.sysmod.last_traceback, self.sysmod.last_value.__traceback__) + self.assertIs(self.sysmod.last_exc, self.sysmod.last_value) + self.assertEqual(traceback.format_exception(self.sysmod.last_exc), [ + 'Traceback (most recent call last):\n', + ' File "", line 1, in \n', + ' File "", line 2, in f\n', + 'ValueError: BOOM!\n']) + + # TODO: RUSTPYTHON + @unittest.expectedFailure + def test_sysexcepthook_syntax_error(self): + self.infunc.side_effect = ["def f():", + " x = ?", + "", EOFError('Finished')] hook = mock.Mock() self.sysmod.excepthook = hook self.console.interact() - self.assertTrue(hook.called) + hook.assert_called() + hook.assert_called_with(self.sysmod.last_type, + self.sysmod.last_value, + self.sysmod.last_traceback) + self.assertIs(self.sysmod.last_type, SyntaxError) + self.assertIs(type(self.sysmod.last_value), SyntaxError) + self.assertIsNone(self.sysmod.last_traceback) + self.assertIsNone(self.sysmod.last_value.__traceback__) + self.assertIs(self.sysmod.last_exc, self.sysmod.last_value) + self.assertEqual(traceback.format_exception(self.sysmod.last_exc), [ + ' File "", line 2\n', + ' x = ?\n', + ' ^\n', + 'SyntaxError: invalid syntax\n']) + + # TODO: RUSTPYTHON + @unittest.expectedFailure + def test_sysexcepthook_indentation_error(self): + self.infunc.side_effect = [" 1", EOFError('Finished')] + hook = mock.Mock() + self.sysmod.excepthook = hook + self.console.interact() + hook.assert_called() + hook.assert_called_with(self.sysmod.last_type, + self.sysmod.last_value, + self.sysmod.last_traceback) + self.assertIs(self.sysmod.last_type, IndentationError) + self.assertIs(type(self.sysmod.last_value), IndentationError) + self.assertIsNone(self.sysmod.last_traceback) + self.assertIsNone(self.sysmod.last_value.__traceback__) + self.assertIs(self.sysmod.last_exc, self.sysmod.last_value) + self.assertEqual(traceback.format_exception(self.sysmod.last_exc), [ + ' File "", line 1\n', + ' 1\n', + 'IndentationError: unexpected indent\n']) + + def test_sysexcepthook_crashing_doesnt_close_repl(self): + self.infunc.side_effect = ["1/0", "a = 123", "print(a)", EOFError('Finished')] + self.sysmod.excepthook = 1 + self.console.interact() + self.assertEqual(['write', ('123', ), {}], self.stdout.method_calls[0]) + error = "".join(call.args[0] for call in self.stderr.method_calls if call[0] == 'write') + self.assertIn("Error in sys.excepthook:", error) + self.assertEqual(error.count("'int' object is not callable"), 1) + self.assertIn("Original exception was:", error) + self.assertIn("division by zero", error) + + def test_sysexcepthook_raising_BaseException(self): + self.infunc.side_effect = ["1/0", "a = 123", "print(a)", EOFError('Finished')] + s = "not so fast" + def raise_base(*args, **kwargs): + raise BaseException(s) + self.sysmod.excepthook = raise_base + self.console.interact() + self.assertEqual(['write', ('123', ), {}], self.stdout.method_calls[0]) + error = "".join(call.args[0] for call in self.stderr.method_calls if call[0] == 'write') + self.assertIn("Error in sys.excepthook:", error) + self.assertEqual(error.count("not so fast"), 1) + self.assertIn("Original exception was:", error) + self.assertIn("division by zero", error) + + def test_sysexcepthook_raising_SystemExit_gets_through(self): + self.infunc.side_effect = ["1/0"] + def raise_base(*args, **kwargs): + raise SystemExit + self.sysmod.excepthook = raise_base + with self.assertRaises(SystemExit): + self.console.interact() def test_banner(self): # with banner @@ -115,6 +261,7 @@ def test_exit_msg(self): expected = message + '\n' self.assertEqual(err_msg, ['write', (expected,), {}]) + # TODO: RUSTPYTHON @unittest.expectedFailure def test_cause_tb(self): @@ -132,9 +279,12 @@ def test_cause_tb(self): ValueError """) self.assertIn(expected, output) + self.assertIs(self.sysmod.last_type, ValueError) + self.assertIs(type(self.sysmod.last_value), ValueError) + self.assertIs(self.sysmod.last_traceback, self.sysmod.last_value.__traceback__) + self.assertIsNotNone(self.sysmod.last_traceback) + self.assertIs(self.sysmod.last_exc, self.sysmod.last_value) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_context_tb(self): self.infunc.side_effect = ["try: ham\nexcept: eggs\n", EOFError('Finished')] @@ -152,6 +302,28 @@ def test_context_tb(self): NameError: name 'eggs' is not defined """) self.assertIn(expected, output) + self.assertIs(self.sysmod.last_type, NameError) + self.assertIs(type(self.sysmod.last_value), NameError) + self.assertIs(self.sysmod.last_traceback, self.sysmod.last_value.__traceback__) + self.assertIsNotNone(self.sysmod.last_traceback) + self.assertIs(self.sysmod.last_exc, self.sysmod.last_value) + + +class TestInteractiveConsoleLocalExit(unittest.TestCase, MockSys): + + def setUp(self): + self.console = code.InteractiveConsole(local_exit=True) + self.mock_sys() + + @unittest.skipIf(sys.flags.no_site, "exit() isn't defined unless there's a site module") + def test_exit(self): + # default exit message + self.infunc.side_effect = ["exit()"] + self.console.interact(banner='') + self.assertEqual(len(self.stderr.method_calls), 2) + err_msg = self.stderr.method_calls[1] + expected = 'now exiting InteractiveConsole...\n' + self.assertEqual(err_msg, ['write', (expected,), {}]) if __name__ == "__main__": From 6bf1ab65c08b614d4d15c716ba84e8dcf2774111 Mon Sep 17 00:00:00 2001 From: "Jeong, YunWon" <69878+youknowone@users.noreply.github.com> Date: Wed, 31 Dec 2025 01:58:34 +0900 Subject: [PATCH 0011/1575] Initiailzer for PyCStructure (#6586) --- crates/vm/src/stdlib/ctypes/structure.rs | 140 ++++++++++++++--------- crates/vm/src/stdlib/ctypes/union.rs | 8 +- 2 files changed, 94 insertions(+), 54 deletions(-) diff --git a/crates/vm/src/stdlib/ctypes/structure.rs b/crates/vm/src/stdlib/ctypes/structure.rs index 295ce0d87cf..732f3c66801 100644 --- a/crates/vm/src/stdlib/ctypes/structure.rs +++ b/crates/vm/src/stdlib/ctypes/structure.rs @@ -368,6 +368,7 @@ impl PyCStructType { // Store StgInfo with aligned size and total alignment let mut stg_info = StgInfo::new(aligned_size, total_align); + stg_info.length = fields.len(); stg_info.format = Some(format); stg_info.flags |= StgInfoFlags::DICTFLAG_FINAL; // Mark as finalized if has_pointer { @@ -511,7 +512,7 @@ impl Debug for PyCStructure { impl Constructor for PyCStructure { type Args = FuncArgs; - fn slot_new(cls: PyTypeRef, args: FuncArgs, vm: &VirtualMachine) -> PyResult { + fn slot_new(cls: PyTypeRef, _args: FuncArgs, vm: &VirtualMachine) -> PyResult { // Check for abstract class and extract values in a block to drop the borrow let (total_size, total_align, length) = { let stg_info = cls.stg_info(vm)?; @@ -523,79 +524,116 @@ impl Constructor for PyCStructure { stg_info_mut.flags |= StgInfoFlags::DICTFLAG_FINAL; } - // Get _fields_ from the class using get_attr to properly search MRO - let fields_attr = cls.as_object().get_attr("_fields_", vm).ok(); + // Initialize buffer with zeros using computed size + let mut new_stg_info = StgInfo::new(total_size, total_align); + new_stg_info.length = length; + PyCStructure(PyCData::from_stg_info(&new_stg_info)) + .into_ref_with_type(vm, cls) + .map(Into::into) + } + + fn py_new(_cls: &Py, _args: Self::Args, _vm: &VirtualMachine) -> PyResult { + unimplemented!("use slot_new") + } +} + +impl PyCStructure { + /// Recursively initialize positional arguments through inheritance chain + /// Returns the number of arguments consumed + fn init_pos_args( + self_obj: &Py, + type_obj: &Py, + args: &[PyObjectRef], + kwargs: &indexmap::IndexMap, + index: usize, + vm: &VirtualMachine, + ) -> PyResult { + let mut current_index = index; - // Collect field names for initialization - let mut field_names: Vec = Vec::new(); - if let Some(fields_attr) = fields_attr { - let fields: Vec = if let Some(list) = fields_attr.downcast_ref::() + // 1. First process base class fields recursively + let base_clone = { + let bases = type_obj.bases.read(); + if let Some(base) = bases.first() + && base.stg_info_opt().is_some() { - list.borrow_vec().to_vec() - } else if let Some(tuple) = fields_attr.downcast_ref::() { - tuple.to_vec() + Some(base.clone()) } else { - vec![] - }; + None + } + }; + + if let Some(ref base) = base_clone { + current_index = Self::init_pos_args(self_obj, base, args, kwargs, current_index, vm)?; + } + + // 2. Process this class's _fields_ + if let Some(fields_attr) = type_obj.get_direct_attr(vm.ctx.intern_str("_fields_")) { + let fields: Vec = fields_attr.try_to_value(vm)?; for field in fields.iter() { - let Some(field_tuple) = field.downcast_ref::() else { - continue; - }; - if field_tuple.len() < 2 { - continue; + if current_index >= args.len() { + break; } - if let Some(name) = field_tuple.first().unwrap().downcast_ref::() { - field_names.push(name.to_string()); + if let Some(tuple) = field.downcast_ref::() + && let Some(name) = tuple.first() + && let Some(name_str) = name.downcast_ref::() + { + let field_name = name_str.as_str().to_owned(); + // Check for duplicate in kwargs + if kwargs.contains_key(&field_name) { + return Err(vm.new_type_error(format!( + "duplicate values for field {:?}", + field_name + ))); + } + self_obj.as_object().set_attr( + vm.ctx.intern_str(field_name), + args[current_index].clone(), + vm, + )?; + current_index += 1; } } } - // Initialize buffer with zeros using computed size - let mut stg_info = StgInfo::new(total_size, total_align); - stg_info.length = if length > 0 { - length - } else { - field_names.len() - }; - stg_info.paramfunc = super::base::ParamFunc::Structure; - let instance = PyCStructure(PyCData::from_stg_info(&stg_info)); + Ok(current_index) + } +} - // Handle keyword arguments for field initialization - let py_instance = instance.into_ref_with_type(vm, cls.clone())?; - let py_obj: PyObjectRef = py_instance.clone().into(); +impl Initializer for PyCStructure { + type Args = FuncArgs; - // Set field values from kwargs using standard attribute setting - for (key, value) in args.kwargs.iter() { - if field_names.iter().any(|n| n == key.as_str()) { - py_obj.set_attr(vm.ctx.intern_str(key.as_str()), value.clone(), vm)?; + fn init(zelf: crate::PyRef, args: Self::Args, vm: &VirtualMachine) -> PyResult<()> { + // Struct_init: handle positional and keyword arguments + let cls = zelf.class().to_owned(); + + // 1. Process positional arguments recursively through inheritance chain + if !args.args.is_empty() { + let consumed = + PyCStructure::init_pos_args(&zelf, &cls, &args.args, &args.kwargs, 0, vm)?; + + if consumed < args.args.len() { + return Err(vm.new_type_error("too many initializers")); } } - // Set field values from positional args - if args.args.len() > field_names.len() { - return Err(vm.new_type_error("too many initializers".to_string())); - } - for (i, value) in args.args.iter().enumerate() { - py_obj.set_attr( - vm.ctx.intern_str(field_names[i].as_str()), - value.clone(), - vm, - )?; + // 2. Process keyword arguments + for (key, value) in args.kwargs.iter() { + zelf.as_object() + .set_attr(vm.ctx.intern_str(key.as_str()), value.clone(), vm)?; } - Ok(py_instance.into()) - } - - fn py_new(_cls: &Py, _args: Self::Args, _vm: &VirtualMachine) -> PyResult { - unimplemented!("use slot_new") + Ok(()) } } // Note: GetAttr and SetAttr are not implemented here. // Field access is handled by CField descriptors registered on the class. -#[pyclass(flags(BASETYPE, IMMUTABLETYPE), with(Constructor, AsBuffer))] +#[pyclass( + flags(BASETYPE, IMMUTABLETYPE), + with(Constructor, Initializer, AsBuffer) +)] impl PyCStructure { #[pygetset] fn _b0_(&self) -> Option { diff --git a/crates/vm/src/stdlib/ctypes/union.rs b/crates/vm/src/stdlib/ctypes/union.rs index 0da1ffee3fd..aecef18ec7c 100644 --- a/crates/vm/src/stdlib/ctypes/union.rs +++ b/crates/vm/src/stdlib/ctypes/union.rs @@ -273,6 +273,7 @@ impl PyCUnionType { // Store StgInfo with aligned size let mut stg_info = StgInfo::new(aligned_size, total_align); + stg_info.length = fields.len(); stg_info.flags |= StgInfoFlags::DICTFLAG_FINAL | StgInfoFlags::TYPEFLAG_HASUNION; // PEP 3118 doesn't support union. Use 'B' for bytes. stg_info.format = Some("B".to_string()); @@ -431,9 +432,9 @@ impl Constructor for PyCUnion { fn slot_new(cls: PyTypeRef, _args: FuncArgs, vm: &VirtualMachine) -> PyResult { // Check for abstract class and extract values in a block to drop the borrow - let (total_size, total_align) = { + let (total_size, total_align, length) = { let stg_info = cls.stg_info(vm)?; - (stg_info.size, stg_info.align) + (stg_info.size, stg_info.align, stg_info.length) }; // Mark the class as finalized (instance creation finalizes the type) @@ -442,7 +443,8 @@ impl Constructor for PyCUnion { } // Initialize buffer with zeros using computed size - let new_stg_info = StgInfo::new(total_size, total_align); + let mut new_stg_info = StgInfo::new(total_size, total_align); + new_stg_info.length = length; PyCUnion(PyCData::from_stg_info(&new_stg_info)) .into_ref_with_type(vm, cls) .map(Into::into) From e22091ef606c4eade1850fb5d57fdf0fc3693965 Mon Sep 17 00:00:00 2001 From: Terry Tianlin Luan <44815845+terryluan12@users.noreply.github.com> Date: Tue, 30 Dec 2025 12:31:08 -0500 Subject: [PATCH 0012/1575] Update straightforward packages Pt 1 (#6595) * Deleted _pycodecs.py * Updated code.py library * Updated the _pydatetime.py lib * Removed distutils package * Updated doctest package * * Updated datetimetester.py * Added back in _pycodecs.py * Used tool to verify datetimetester + test_code_module --- Lib/_pydatetime.py | 62 +- Lib/distutils/README | 13 - Lib/distutils/__init__.py | 13 - Lib/distutils/_msvccompiler.py | 574 ---------- Lib/distutils/archive_util.py | 256 ----- Lib/distutils/bcppcompiler.py | 393 ------- Lib/distutils/ccompiler.py | 1115 ------------------- Lib/distutils/cmd.py | 434 -------- Lib/distutils/command/__init__.py | 31 - Lib/distutils/command/bdist.py | 143 --- Lib/distutils/command/bdist_dumb.py | 123 -- Lib/distutils/command/bdist_msi.py | 741 ------------ Lib/distutils/command/bdist_rpm.py | 582 ---------- Lib/distutils/command/bdist_wininst.py | 367 ------ Lib/distutils/command/build.py | 157 --- Lib/distutils/command/build_clib.py | 209 ---- Lib/distutils/command/build_ext.py | 755 ------------- Lib/distutils/command/build_py.py | 416 ------- Lib/distutils/command/build_scripts.py | 160 --- Lib/distutils/command/check.py | 145 --- Lib/distutils/command/clean.py | 76 -- Lib/distutils/command/command_template | 33 - Lib/distutils/command/config.py | 347 ------ Lib/distutils/command/install.py | 705 ------------ Lib/distutils/command/install_data.py | 79 -- Lib/distutils/command/install_egg_info.py | 97 -- Lib/distutils/command/install_headers.py | 47 - Lib/distutils/command/install_lib.py | 221 ---- Lib/distutils/command/install_scripts.py | 60 - Lib/distutils/command/register.py | 304 ----- Lib/distutils/command/sdist.py | 456 -------- Lib/distutils/command/upload.py | 200 ---- Lib/distutils/config.py | 131 --- Lib/distutils/core.py | 234 ---- Lib/distutils/cygwinccompiler.py | 405 ------- Lib/distutils/debug.py | 5 - Lib/distutils/dep_util.py | 92 -- Lib/distutils/dir_util.py | 223 ---- Lib/distutils/dist.py | 1236 --------------------- Lib/distutils/errors.py | 97 -- Lib/distutils/extension.py | 240 ---- Lib/distutils/fancy_getopt.py | 457 -------- Lib/distutils/file_util.py | 238 ---- Lib/distutils/filelist.py | 327 ------ Lib/distutils/log.py | 77 -- Lib/distutils/msvc9compiler.py | 791 ------------- Lib/distutils/msvccompiler.py | 643 ----------- Lib/distutils/spawn.py | 192 ---- Lib/distutils/sysconfig.py | 556 --------- Lib/distutils/text_file.py | 286 ----- Lib/distutils/unixccompiler.py | 333 ------ Lib/distutils/util.py | 557 ---------- Lib/distutils/version.py | 343 ------ Lib/distutils/versionpredicate.py | 166 --- Lib/doctest.py | 292 +++-- Lib/test/datetimetester.py | 511 +++++++-- Lib/test/test_code_module.py | 8 +- 57 files changed, 638 insertions(+), 17116 deletions(-) delete mode 100644 Lib/distutils/README delete mode 100644 Lib/distutils/__init__.py delete mode 100644 Lib/distutils/_msvccompiler.py delete mode 100644 Lib/distutils/archive_util.py delete mode 100644 Lib/distutils/bcppcompiler.py delete mode 100644 Lib/distutils/ccompiler.py delete mode 100644 Lib/distutils/cmd.py delete mode 100644 Lib/distutils/command/__init__.py delete mode 100644 Lib/distutils/command/bdist.py delete mode 100644 Lib/distutils/command/bdist_dumb.py delete mode 100644 Lib/distutils/command/bdist_msi.py delete mode 100644 Lib/distutils/command/bdist_rpm.py delete mode 100644 Lib/distutils/command/bdist_wininst.py delete mode 100644 Lib/distutils/command/build.py delete mode 100644 Lib/distutils/command/build_clib.py delete mode 100644 Lib/distutils/command/build_ext.py delete mode 100644 Lib/distutils/command/build_py.py delete mode 100644 Lib/distutils/command/build_scripts.py delete mode 100644 Lib/distutils/command/check.py delete mode 100644 Lib/distutils/command/clean.py delete mode 100644 Lib/distutils/command/command_template delete mode 100644 Lib/distutils/command/config.py delete mode 100644 Lib/distutils/command/install.py delete mode 100644 Lib/distutils/command/install_data.py delete mode 100644 Lib/distutils/command/install_egg_info.py delete mode 100644 Lib/distutils/command/install_headers.py delete mode 100644 Lib/distutils/command/install_lib.py delete mode 100644 Lib/distutils/command/install_scripts.py delete mode 100644 Lib/distutils/command/register.py delete mode 100644 Lib/distutils/command/sdist.py delete mode 100644 Lib/distutils/command/upload.py delete mode 100644 Lib/distutils/config.py delete mode 100644 Lib/distutils/core.py delete mode 100644 Lib/distutils/cygwinccompiler.py delete mode 100644 Lib/distutils/debug.py delete mode 100644 Lib/distutils/dep_util.py delete mode 100644 Lib/distutils/dir_util.py delete mode 100644 Lib/distutils/dist.py delete mode 100644 Lib/distutils/errors.py delete mode 100644 Lib/distutils/extension.py delete mode 100644 Lib/distutils/fancy_getopt.py delete mode 100644 Lib/distutils/file_util.py delete mode 100644 Lib/distutils/filelist.py delete mode 100644 Lib/distutils/log.py delete mode 100644 Lib/distutils/msvc9compiler.py delete mode 100644 Lib/distutils/msvccompiler.py delete mode 100644 Lib/distutils/spawn.py delete mode 100644 Lib/distutils/sysconfig.py delete mode 100644 Lib/distutils/text_file.py delete mode 100644 Lib/distutils/unixccompiler.py delete mode 100644 Lib/distutils/util.py delete mode 100644 Lib/distutils/version.py delete mode 100644 Lib/distutils/versionpredicate.py diff --git a/Lib/_pydatetime.py b/Lib/_pydatetime.py index cd0ea900bfb..38e1f764f00 100644 --- a/Lib/_pydatetime.py +++ b/Lib/_pydatetime.py @@ -402,6 +402,8 @@ def _parse_hh_mm_ss_ff(tstr): raise ValueError("Invalid microsecond component") else: pos += 1 + if not all(map(_is_ascii_digit, tstr[pos:])): + raise ValueError("Non-digit values in fraction") len_remainder = len_str - pos @@ -413,9 +415,6 @@ def _parse_hh_mm_ss_ff(tstr): time_comps[3] = int(tstr[pos:(pos+to_parse)]) if to_parse < 6: time_comps[3] *= _FRACTION_CORRECTION[to_parse-1] - if (len_remainder > to_parse - and not all(map(_is_ascii_digit, tstr[(pos+to_parse):]))): - raise ValueError("Non-digit values in unparsed fraction") return time_comps @@ -556,10 +555,6 @@ def _check_tzinfo_arg(tz): if tz is not None and not isinstance(tz, tzinfo): raise TypeError("tzinfo argument must be None or of a tzinfo subclass") -def _cmperror(x, y): - raise TypeError("can't compare '%s' to '%s'" % ( - type(x).__name__, type(y).__name__)) - def _divide_and_round(a, b): """divide a by b and round result to the nearest integer @@ -970,6 +965,8 @@ def __new__(cls, year, month=None, day=None): @classmethod def fromtimestamp(cls, t): "Construct a date from a POSIX timestamp (like time.time())." + if t is None: + raise TypeError("'NoneType' object cannot be interpreted as an integer") y, m, d, hh, mm, ss, weekday, jday, dst = _time.localtime(t) return cls(y, m, d) @@ -1059,8 +1056,8 @@ def isoformat(self): This is 'YYYY-MM-DD'. References: - - http://www.w3.org/TR/NOTE-datetime - - http://www.cl.cam.ac.uk/~mgk25/iso-time.html + - https://www.w3.org/TR/NOTE-datetime + - https://www.cl.cam.ac.uk/~mgk25/iso-time.html """ return "%04d-%02d-%02d" % (self._year, self._month, self._day) @@ -1108,35 +1105,38 @@ def replace(self, year=None, month=None, day=None): day = self._day return type(self)(year, month, day) + __replace__ = replace + # Comparisons of date objects with other. def __eq__(self, other): - if isinstance(other, date): + if isinstance(other, date) and not isinstance(other, datetime): return self._cmp(other) == 0 return NotImplemented def __le__(self, other): - if isinstance(other, date): + if isinstance(other, date) and not isinstance(other, datetime): return self._cmp(other) <= 0 return NotImplemented def __lt__(self, other): - if isinstance(other, date): + if isinstance(other, date) and not isinstance(other, datetime): return self._cmp(other) < 0 return NotImplemented def __ge__(self, other): - if isinstance(other, date): + if isinstance(other, date) and not isinstance(other, datetime): return self._cmp(other) >= 0 return NotImplemented def __gt__(self, other): - if isinstance(other, date): + if isinstance(other, date) and not isinstance(other, datetime): return self._cmp(other) > 0 return NotImplemented def _cmp(self, other): assert isinstance(other, date) + assert not isinstance(other, datetime) y, m, d = self._year, self._month, self._day y2, m2, d2 = other._year, other._month, other._day return _cmp((y, m, d), (y2, m2, d2)) @@ -1191,7 +1191,7 @@ def isocalendar(self): The first week is 1; Monday is 1 ... Sunday is 7. ISO calendar algorithm taken from - http://www.phys.uu.nl/~vgent/calendar/isocalendar.htm + https://www.phys.uu.nl/~vgent/calendar/isocalendar.htm (used with permission) """ year = self._year @@ -1633,6 +1633,8 @@ def replace(self, hour=None, minute=None, second=None, microsecond=None, fold = self._fold return type(self)(hour, minute, second, microsecond, tzinfo, fold=fold) + __replace__ = replace + # Pickle support. def _getstate(self, protocol=3): @@ -1680,7 +1682,7 @@ class datetime(date): The year, month and day arguments are required. tzinfo may be None, or an instance of a tzinfo subclass. The remaining arguments may be ints. """ - __slots__ = date.__slots__ + time.__slots__ + __slots__ = time.__slots__ def __new__(cls, year, month=None, day=None, hour=0, minute=0, second=0, microsecond=0, tzinfo=None, *, fold=0): @@ -1979,6 +1981,8 @@ def replace(self, year=None, month=None, day=None, hour=None, return type(self)(year, month, day, hour, minute, second, microsecond, tzinfo, fold=fold) + __replace__ = replace + def _local_timezone(self): if self.tzinfo is None: ts = self._mktime() @@ -2040,7 +2044,7 @@ def isoformat(self, sep='T', timespec='auto'): By default, the fractional part is omitted if self.microsecond == 0. If self.tzinfo is not None, the UTC offset is also attached, giving - giving a full format of 'YYYY-MM-DD HH:MM:SS.mmmmmm+HH:MM'. + a full format of 'YYYY-MM-DD HH:MM:SS.mmmmmm+HH:MM'. Optional argument sep specifies the separator between date and time, default 'T'. @@ -2131,42 +2135,32 @@ def dst(self): def __eq__(self, other): if isinstance(other, datetime): return self._cmp(other, allow_mixed=True) == 0 - elif not isinstance(other, date): - return NotImplemented else: - return False + return NotImplemented def __le__(self, other): if isinstance(other, datetime): return self._cmp(other) <= 0 - elif not isinstance(other, date): - return NotImplemented else: - _cmperror(self, other) + return NotImplemented def __lt__(self, other): if isinstance(other, datetime): return self._cmp(other) < 0 - elif not isinstance(other, date): - return NotImplemented else: - _cmperror(self, other) + return NotImplemented def __ge__(self, other): if isinstance(other, datetime): return self._cmp(other) >= 0 - elif not isinstance(other, date): - return NotImplemented else: - _cmperror(self, other) + return NotImplemented def __gt__(self, other): if isinstance(other, datetime): return self._cmp(other) > 0 - elif not isinstance(other, date): - return NotImplemented else: - _cmperror(self, other) + return NotImplemented def _cmp(self, other, allow_mixed=False): assert isinstance(other, datetime) @@ -2311,7 +2305,6 @@ def __reduce__(self): def _isoweek1monday(year): # Helper to calculate the day number of the Monday starting week 1 - # XXX This could be done more efficiently THURSDAY = 3 firstday = _ymd2ord(year, 1, 1) firstweekday = (firstday + 6) % 7 # See weekday() above @@ -2341,6 +2334,9 @@ def __new__(cls, offset, name=_Omitted): "timedelta(hours=24).") return cls._create(offset, name) + def __init_subclass__(cls): + raise TypeError("type 'datetime.timezone' is not an acceptable base type") + @classmethod def _create(cls, offset, name=None): self = tzinfo.__new__(cls) diff --git a/Lib/distutils/README b/Lib/distutils/README deleted file mode 100644 index 408a203b85d..00000000000 --- a/Lib/distutils/README +++ /dev/null @@ -1,13 +0,0 @@ -This directory contains the Distutils package. - -There's a full documentation available at: - - http://docs.python.org/distutils/ - -The Distutils-SIG web page is also a good starting point: - - http://www.python.org/sigs/distutils-sig/ - -WARNING : Distutils must remain compatible with 2.3 - -$Id$ diff --git a/Lib/distutils/__init__.py b/Lib/distutils/__init__.py deleted file mode 100644 index d823d040a1c..00000000000 --- a/Lib/distutils/__init__.py +++ /dev/null @@ -1,13 +0,0 @@ -"""distutils - -The main package for the Python Module Distribution Utilities. Normally -used from a setup script as - - from distutils.core import setup - - setup (...) -""" - -import sys - -__version__ = sys.version[:sys.version.index(' ')] diff --git a/Lib/distutils/_msvccompiler.py b/Lib/distutils/_msvccompiler.py deleted file mode 100644 index 30b3b473985..00000000000 --- a/Lib/distutils/_msvccompiler.py +++ /dev/null @@ -1,574 +0,0 @@ -"""distutils._msvccompiler - -Contains MSVCCompiler, an implementation of the abstract CCompiler class -for Microsoft Visual Studio 2015. - -The module is compatible with VS 2015 and later. You can find legacy support -for older versions in distutils.msvc9compiler and distutils.msvccompiler. -""" - -# Written by Perry Stoll -# hacked by Robin Becker and Thomas Heller to do a better job of -# finding DevStudio (through the registry) -# ported to VS 2005 and VS 2008 by Christian Heimes -# ported to VS 2015 by Steve Dower - -import os -import shutil -import stat -import subprocess -import winreg - -from distutils.errors import DistutilsExecError, DistutilsPlatformError, \ - CompileError, LibError, LinkError -from distutils.ccompiler import CCompiler, gen_lib_options -from distutils import log -from distutils.util import get_platform - -from itertools import count - -def _find_vc2015(): - try: - key = winreg.OpenKeyEx( - winreg.HKEY_LOCAL_MACHINE, - r"Software\Microsoft\VisualStudio\SxS\VC7", - access=winreg.KEY_READ | winreg.KEY_WOW64_32KEY - ) - except OSError: - log.debug("Visual C++ is not registered") - return None, None - - best_version = 0 - best_dir = None - with key: - for i in count(): - try: - v, vc_dir, vt = winreg.EnumValue(key, i) - except OSError: - break - if v and vt == winreg.REG_SZ and os.path.isdir(vc_dir): - try: - version = int(float(v)) - except (ValueError, TypeError): - continue - if version >= 14 and version > best_version: - best_version, best_dir = version, vc_dir - return best_version, best_dir - -def _find_vc2017(): - import _distutils_findvs - import threading - - best_version = 0, # tuple for full version comparisons - best_dir = None - - # We need to call findall() on its own thread because it will - # initialize COM. - all_packages = [] - def _getall(): - all_packages.extend(_distutils_findvs.findall()) - t = threading.Thread(target=_getall) - t.start() - t.join() - - for name, version_str, path, packages in all_packages: - if 'Microsoft.VisualStudio.Component.VC.Tools.x86.x64' in packages: - vc_dir = os.path.join(path, 'VC', 'Auxiliary', 'Build') - if not os.path.isdir(vc_dir): - continue - try: - version = tuple(int(i) for i in version_str.split('.')) - except (ValueError, TypeError): - continue - if version > best_version: - best_version, best_dir = version, vc_dir - try: - best_version = best_version[0] - except IndexError: - best_version = None - return best_version, best_dir - -def _find_vcvarsall(plat_spec): - best_version, best_dir = _find_vc2017() - vcruntime = None - vcruntime_plat = 'x64' if 'amd64' in plat_spec else 'x86' - if best_version: - vcredist = os.path.join(best_dir, "..", "..", "redist", "MSVC", "**", - "Microsoft.VC141.CRT", "vcruntime140.dll") - try: - import glob - vcruntime = glob.glob(vcredist, recursive=True)[-1] - except (ImportError, OSError, LookupError): - vcruntime = None - - if not best_version: - best_version, best_dir = _find_vc2015() - if best_version: - vcruntime = os.path.join(best_dir, 'redist', vcruntime_plat, - "Microsoft.VC140.CRT", "vcruntime140.dll") - - if not best_version: - log.debug("No suitable Visual C++ version found") - return None, None - - vcvarsall = os.path.join(best_dir, "vcvarsall.bat") - if not os.path.isfile(vcvarsall): - log.debug("%s cannot be found", vcvarsall) - return None, None - - if not vcruntime or not os.path.isfile(vcruntime): - log.debug("%s cannot be found", vcruntime) - vcruntime = None - - return vcvarsall, vcruntime - -def _get_vc_env(plat_spec): - if os.getenv("DISTUTILS_USE_SDK"): - return { - key.lower(): value - for key, value in os.environ.items() - } - - vcvarsall, vcruntime = _find_vcvarsall(plat_spec) - if not vcvarsall: - raise DistutilsPlatformError("Unable to find vcvarsall.bat") - - try: - out = subprocess.check_output( - 'cmd /u /c "{}" {} && set'.format(vcvarsall, plat_spec), - stderr=subprocess.STDOUT, - ).decode('utf-16le', errors='replace') - except subprocess.CalledProcessError as exc: - log.error(exc.output) - raise DistutilsPlatformError("Error executing {}" - .format(exc.cmd)) - - env = { - key.lower(): value - for key, _, value in - (line.partition('=') for line in out.splitlines()) - if key and value - } - - if vcruntime: - env['py_vcruntime_redist'] = vcruntime - return env - -def _find_exe(exe, paths=None): - """Return path to an MSVC executable program. - - Tries to find the program in several places: first, one of the - MSVC program search paths from the registry; next, the directories - in the PATH environment variable. If any of those work, return an - absolute path that is known to exist. If none of them work, just - return the original program name, 'exe'. - """ - if not paths: - paths = os.getenv('path').split(os.pathsep) - for p in paths: - fn = os.path.join(os.path.abspath(p), exe) - if os.path.isfile(fn): - return fn - return exe - -# A map keyed by get_platform() return values to values accepted by -# 'vcvarsall.bat'. Always cross-compile from x86 to work with the -# lighter-weight MSVC installs that do not include native 64-bit tools. -PLAT_TO_VCVARS = { - 'win32' : 'x86', - 'win-amd64' : 'x86_amd64', -} - -# A set containing the DLLs that are guaranteed to be available for -# all micro versions of this Python version. Known extension -# dependencies that are not in this set will be copied to the output -# path. -_BUNDLED_DLLS = frozenset(['vcruntime140.dll']) - -class MSVCCompiler(CCompiler) : - """Concrete class that implements an interface to Microsoft Visual C++, - as defined by the CCompiler abstract class.""" - - compiler_type = 'msvc' - - # Just set this so CCompiler's constructor doesn't barf. We currently - # don't use the 'set_executables()' bureaucracy provided by CCompiler, - # as it really isn't necessary for this sort of single-compiler class. - # Would be nice to have a consistent interface with UnixCCompiler, - # though, so it's worth thinking about. - executables = {} - - # Private class data (need to distinguish C from C++ source for compiler) - _c_extensions = ['.c'] - _cpp_extensions = ['.cc', '.cpp', '.cxx'] - _rc_extensions = ['.rc'] - _mc_extensions = ['.mc'] - - # Needed for the filename generation methods provided by the - # base class, CCompiler. - src_extensions = (_c_extensions + _cpp_extensions + - _rc_extensions + _mc_extensions) - res_extension = '.res' - obj_extension = '.obj' - static_lib_extension = '.lib' - shared_lib_extension = '.dll' - static_lib_format = shared_lib_format = '%s%s' - exe_extension = '.exe' - - - def __init__(self, verbose=0, dry_run=0, force=0): - CCompiler.__init__ (self, verbose, dry_run, force) - # target platform (.plat_name is consistent with 'bdist') - self.plat_name = None - self.initialized = False - - def initialize(self, plat_name=None): - # multi-init means we would need to check platform same each time... - assert not self.initialized, "don't init multiple times" - if plat_name is None: - plat_name = get_platform() - # sanity check for platforms to prevent obscure errors later. - if plat_name not in PLAT_TO_VCVARS: - raise DistutilsPlatformError("--plat-name must be one of {}" - .format(tuple(PLAT_TO_VCVARS))) - - # Get the vcvarsall.bat spec for the requested platform. - plat_spec = PLAT_TO_VCVARS[plat_name] - - vc_env = _get_vc_env(plat_spec) - if not vc_env: - raise DistutilsPlatformError("Unable to find a compatible " - "Visual Studio installation.") - - self._paths = vc_env.get('path', '') - paths = self._paths.split(os.pathsep) - self.cc = _find_exe("cl.exe", paths) - self.linker = _find_exe("link.exe", paths) - self.lib = _find_exe("lib.exe", paths) - self.rc = _find_exe("rc.exe", paths) # resource compiler - self.mc = _find_exe("mc.exe", paths) # message compiler - self.mt = _find_exe("mt.exe", paths) # message compiler - self._vcruntime_redist = vc_env.get('py_vcruntime_redist', '') - - for dir in vc_env.get('include', '').split(os.pathsep): - if dir: - self.add_include_dir(dir.rstrip(os.sep)) - - for dir in vc_env.get('lib', '').split(os.pathsep): - if dir: - self.add_library_dir(dir.rstrip(os.sep)) - - self.preprocess_options = None - # If vcruntime_redist is available, link against it dynamically. Otherwise, - # use /MT[d] to build statically, then switch from libucrt[d].lib to ucrt[d].lib - # later to dynamically link to ucrtbase but not vcruntime. - self.compile_options = [ - '/nologo', '/Ox', '/W3', '/GL', '/DNDEBUG' - ] - self.compile_options.append('/MD' if self._vcruntime_redist else '/MT') - - self.compile_options_debug = [ - '/nologo', '/Od', '/MDd', '/Zi', '/W3', '/D_DEBUG' - ] - - ldflags = [ - '/nologo', '/INCREMENTAL:NO', '/LTCG' - ] - if not self._vcruntime_redist: - ldflags.extend(('/nodefaultlib:libucrt.lib', 'ucrt.lib')) - - ldflags_debug = [ - '/nologo', '/INCREMENTAL:NO', '/LTCG', '/DEBUG:FULL' - ] - - self.ldflags_exe = [*ldflags, '/MANIFEST:EMBED,ID=1'] - self.ldflags_exe_debug = [*ldflags_debug, '/MANIFEST:EMBED,ID=1'] - self.ldflags_shared = [*ldflags, '/DLL', '/MANIFEST:EMBED,ID=2', '/MANIFESTUAC:NO'] - self.ldflags_shared_debug = [*ldflags_debug, '/DLL', '/MANIFEST:EMBED,ID=2', '/MANIFESTUAC:NO'] - self.ldflags_static = [*ldflags] - self.ldflags_static_debug = [*ldflags_debug] - - self._ldflags = { - (CCompiler.EXECUTABLE, None): self.ldflags_exe, - (CCompiler.EXECUTABLE, False): self.ldflags_exe, - (CCompiler.EXECUTABLE, True): self.ldflags_exe_debug, - (CCompiler.SHARED_OBJECT, None): self.ldflags_shared, - (CCompiler.SHARED_OBJECT, False): self.ldflags_shared, - (CCompiler.SHARED_OBJECT, True): self.ldflags_shared_debug, - (CCompiler.SHARED_LIBRARY, None): self.ldflags_static, - (CCompiler.SHARED_LIBRARY, False): self.ldflags_static, - (CCompiler.SHARED_LIBRARY, True): self.ldflags_static_debug, - } - - self.initialized = True - - # -- Worker methods ------------------------------------------------ - - def object_filenames(self, - source_filenames, - strip_dir=0, - output_dir=''): - ext_map = { - **{ext: self.obj_extension for ext in self.src_extensions}, - **{ext: self.res_extension for ext in self._rc_extensions + self._mc_extensions}, - } - - output_dir = output_dir or '' - - def make_out_path(p): - base, ext = os.path.splitext(p) - if strip_dir: - base = os.path.basename(base) - else: - _, base = os.path.splitdrive(base) - if base.startswith((os.path.sep, os.path.altsep)): - base = base[1:] - try: - # XXX: This may produce absurdly long paths. We should check - # the length of the result and trim base until we fit within - # 260 characters. - return os.path.join(output_dir, base + ext_map[ext]) - except LookupError: - # Better to raise an exception instead of silently continuing - # and later complain about sources and targets having - # different lengths - raise CompileError("Don't know how to compile {}".format(p)) - - return list(map(make_out_path, source_filenames)) - - - def compile(self, sources, - output_dir=None, macros=None, include_dirs=None, debug=0, - extra_preargs=None, extra_postargs=None, depends=None): - - if not self.initialized: - self.initialize() - compile_info = self._setup_compile(output_dir, macros, include_dirs, - sources, depends, extra_postargs) - macros, objects, extra_postargs, pp_opts, build = compile_info - - compile_opts = extra_preargs or [] - compile_opts.append('/c') - if debug: - compile_opts.extend(self.compile_options_debug) - else: - compile_opts.extend(self.compile_options) - - - add_cpp_opts = False - - for obj in objects: - try: - src, ext = build[obj] - except KeyError: - continue - if debug: - # pass the full pathname to MSVC in debug mode, - # this allows the debugger to find the source file - # without asking the user to browse for it - src = os.path.abspath(src) - - if ext in self._c_extensions: - input_opt = "/Tc" + src - elif ext in self._cpp_extensions: - input_opt = "/Tp" + src - add_cpp_opts = True - elif ext in self._rc_extensions: - # compile .RC to .RES file - input_opt = src - output_opt = "/fo" + obj - try: - self.spawn([self.rc] + pp_opts + [output_opt, input_opt]) - except DistutilsExecError as msg: - raise CompileError(msg) - continue - elif ext in self._mc_extensions: - # Compile .MC to .RC file to .RES file. - # * '-h dir' specifies the directory for the - # generated include file - # * '-r dir' specifies the target directory of the - # generated RC file and the binary message resource - # it includes - # - # For now (since there are no options to change this), - # we use the source-directory for the include file and - # the build directory for the RC file and message - # resources. This works at least for win32all. - h_dir = os.path.dirname(src) - rc_dir = os.path.dirname(obj) - try: - # first compile .MC to .RC and .H file - self.spawn([self.mc, '-h', h_dir, '-r', rc_dir, src]) - base, _ = os.path.splitext(os.path.basename (src)) - rc_file = os.path.join(rc_dir, base + '.rc') - # then compile .RC to .RES file - self.spawn([self.rc, "/fo" + obj, rc_file]) - - except DistutilsExecError as msg: - raise CompileError(msg) - continue - else: - # how to handle this file? - raise CompileError("Don't know how to compile {} to {}" - .format(src, obj)) - - args = [self.cc] + compile_opts + pp_opts - if add_cpp_opts: - args.append('/EHsc') - args.append(input_opt) - args.append("/Fo" + obj) - args.extend(extra_postargs) - - try: - self.spawn(args) - except DistutilsExecError as msg: - raise CompileError(msg) - - return objects - - - def create_static_lib(self, - objects, - output_libname, - output_dir=None, - debug=0, - target_lang=None): - - if not self.initialized: - self.initialize() - objects, output_dir = self._fix_object_args(objects, output_dir) - output_filename = self.library_filename(output_libname, - output_dir=output_dir) - - if self._need_link(objects, output_filename): - lib_args = objects + ['/OUT:' + output_filename] - if debug: - pass # XXX what goes here? - try: - log.debug('Executing "%s" %s', self.lib, ' '.join(lib_args)) - self.spawn([self.lib] + lib_args) - except DistutilsExecError as msg: - raise LibError(msg) - else: - log.debug("skipping %s (up-to-date)", output_filename) - - - def link(self, - target_desc, - objects, - output_filename, - output_dir=None, - libraries=None, - library_dirs=None, - runtime_library_dirs=None, - export_symbols=None, - debug=0, - extra_preargs=None, - extra_postargs=None, - build_temp=None, - target_lang=None): - - if not self.initialized: - self.initialize() - objects, output_dir = self._fix_object_args(objects, output_dir) - fixed_args = self._fix_lib_args(libraries, library_dirs, - runtime_library_dirs) - libraries, library_dirs, runtime_library_dirs = fixed_args - - if runtime_library_dirs: - self.warn("I don't know what to do with 'runtime_library_dirs': " - + str(runtime_library_dirs)) - - lib_opts = gen_lib_options(self, - library_dirs, runtime_library_dirs, - libraries) - if output_dir is not None: - output_filename = os.path.join(output_dir, output_filename) - - if self._need_link(objects, output_filename): - ldflags = self._ldflags[target_desc, debug] - - export_opts = ["/EXPORT:" + sym for sym in (export_symbols or [])] - - ld_args = (ldflags + lib_opts + export_opts + - objects + ['/OUT:' + output_filename]) - - # The MSVC linker generates .lib and .exp files, which cannot be - # suppressed by any linker switches. The .lib files may even be - # needed! Make sure they are generated in the temporary build - # directory. Since they have different names for debug and release - # builds, they can go into the same directory. - build_temp = os.path.dirname(objects[0]) - if export_symbols is not None: - (dll_name, dll_ext) = os.path.splitext( - os.path.basename(output_filename)) - implib_file = os.path.join( - build_temp, - self.library_filename(dll_name)) - ld_args.append ('/IMPLIB:' + implib_file) - - if extra_preargs: - ld_args[:0] = extra_preargs - if extra_postargs: - ld_args.extend(extra_postargs) - - output_dir = os.path.dirname(os.path.abspath(output_filename)) - self.mkpath(output_dir) - try: - log.debug('Executing "%s" %s', self.linker, ' '.join(ld_args)) - self.spawn([self.linker] + ld_args) - self._copy_vcruntime(output_dir) - except DistutilsExecError as msg: - raise LinkError(msg) - else: - log.debug("skipping %s (up-to-date)", output_filename) - - def _copy_vcruntime(self, output_dir): - vcruntime = self._vcruntime_redist - if not vcruntime or not os.path.isfile(vcruntime): - return - - if os.path.basename(vcruntime).lower() in _BUNDLED_DLLS: - return - - log.debug('Copying "%s"', vcruntime) - vcruntime = shutil.copy(vcruntime, output_dir) - os.chmod(vcruntime, stat.S_IWRITE) - - def spawn(self, cmd): - old_path = os.getenv('path') - try: - os.environ['path'] = self._paths - return super().spawn(cmd) - finally: - os.environ['path'] = old_path - - # -- Miscellaneous methods ----------------------------------------- - # These are all used by the 'gen_lib_options() function, in - # ccompiler.py. - - def library_dir_option(self, dir): - return "/LIBPATH:" + dir - - def runtime_library_dir_option(self, dir): - raise DistutilsPlatformError( - "don't know how to set runtime library search path for MSVC") - - def library_option(self, lib): - return self.library_filename(lib) - - def find_library_file(self, dirs, lib, debug=0): - # Prefer a debugging library if found (and requested), but deal - # with it if we don't have one. - if debug: - try_names = [lib + "_d", lib] - else: - try_names = [lib] - for dir in dirs: - for name in try_names: - libfile = os.path.join(dir, self.library_filename(name)) - if os.path.isfile(libfile): - return libfile - else: - # Oops, didn't find it in *any* of 'dirs' - return None diff --git a/Lib/distutils/archive_util.py b/Lib/distutils/archive_util.py deleted file mode 100644 index b002dc3b845..00000000000 --- a/Lib/distutils/archive_util.py +++ /dev/null @@ -1,256 +0,0 @@ -"""distutils.archive_util - -Utility functions for creating archive files (tarballs, zip files, -that sort of thing).""" - -import os -from warnings import warn -import sys - -try: - import zipfile -except ImportError: - zipfile = None - - -from distutils.errors import DistutilsExecError -from distutils.spawn import spawn -from distutils.dir_util import mkpath -from distutils import log - -try: - from pwd import getpwnam -except ImportError: - getpwnam = None - -try: - from grp import getgrnam -except ImportError: - getgrnam = None - -def _get_gid(name): - """Returns a gid, given a group name.""" - if getgrnam is None or name is None: - return None - try: - result = getgrnam(name) - except KeyError: - result = None - if result is not None: - return result[2] - return None - -def _get_uid(name): - """Returns an uid, given a user name.""" - if getpwnam is None or name is None: - return None - try: - result = getpwnam(name) - except KeyError: - result = None - if result is not None: - return result[2] - return None - -def make_tarball(base_name, base_dir, compress="gzip", verbose=0, dry_run=0, - owner=None, group=None): - """Create a (possibly compressed) tar file from all the files under - 'base_dir'. - - 'compress' must be "gzip" (the default), "bzip2", "xz", "compress", or - None. ("compress" will be deprecated in Python 3.2) - - 'owner' and 'group' can be used to define an owner and a group for the - archive that is being built. If not provided, the current owner and group - will be used. - - The output tar file will be named 'base_dir' + ".tar", possibly plus - the appropriate compression extension (".gz", ".bz2", ".xz" or ".Z"). - - Returns the output filename. - """ - tar_compression = {'gzip': 'gz', 'bzip2': 'bz2', 'xz': 'xz', None: '', - 'compress': ''} - compress_ext = {'gzip': '.gz', 'bzip2': '.bz2', 'xz': '.xz', - 'compress': '.Z'} - - # flags for compression program, each element of list will be an argument - if compress is not None and compress not in compress_ext.keys(): - raise ValueError( - "bad value for 'compress': must be None, 'gzip', 'bzip2', " - "'xz' or 'compress'") - - archive_name = base_name + '.tar' - if compress != 'compress': - archive_name += compress_ext.get(compress, '') - - mkpath(os.path.dirname(archive_name), dry_run=dry_run) - - # creating the tarball - import tarfile # late import so Python build itself doesn't break - - log.info('Creating tar archive') - - uid = _get_uid(owner) - gid = _get_gid(group) - - def _set_uid_gid(tarinfo): - if gid is not None: - tarinfo.gid = gid - tarinfo.gname = group - if uid is not None: - tarinfo.uid = uid - tarinfo.uname = owner - return tarinfo - - if not dry_run: - tar = tarfile.open(archive_name, 'w|%s' % tar_compression[compress]) - try: - tar.add(base_dir, filter=_set_uid_gid) - finally: - tar.close() - - # compression using `compress` - if compress == 'compress': - warn("'compress' will be deprecated.", PendingDeprecationWarning) - # the option varies depending on the platform - compressed_name = archive_name + compress_ext[compress] - if sys.platform == 'win32': - cmd = [compress, archive_name, compressed_name] - else: - cmd = [compress, '-f', archive_name] - spawn(cmd, dry_run=dry_run) - return compressed_name - - return archive_name - -def make_zipfile(base_name, base_dir, verbose=0, dry_run=0): - """Create a zip file from all the files under 'base_dir'. - - The output zip file will be named 'base_name' + ".zip". Uses either the - "zipfile" Python module (if available) or the InfoZIP "zip" utility - (if installed and found on the default search path). If neither tool is - available, raises DistutilsExecError. Returns the name of the output zip - file. - """ - zip_filename = base_name + ".zip" - mkpath(os.path.dirname(zip_filename), dry_run=dry_run) - - # If zipfile module is not available, try spawning an external - # 'zip' command. - if zipfile is None: - if verbose: - zipoptions = "-r" - else: - zipoptions = "-rq" - - try: - spawn(["zip", zipoptions, zip_filename, base_dir], - dry_run=dry_run) - except DistutilsExecError: - # XXX really should distinguish between "couldn't find - # external 'zip' command" and "zip failed". - raise DistutilsExecError(("unable to create zip file '%s': " - "could neither import the 'zipfile' module nor " - "find a standalone zip utility") % zip_filename) - - else: - log.info("creating '%s' and adding '%s' to it", - zip_filename, base_dir) - - if not dry_run: - try: - zip = zipfile.ZipFile(zip_filename, "w", - compression=zipfile.ZIP_DEFLATED) - except RuntimeError: - zip = zipfile.ZipFile(zip_filename, "w", - compression=zipfile.ZIP_STORED) - - if base_dir != os.curdir: - path = os.path.normpath(os.path.join(base_dir, '')) - zip.write(path, path) - log.info("adding '%s'", path) - for dirpath, dirnames, filenames in os.walk(base_dir): - for name in dirnames: - path = os.path.normpath(os.path.join(dirpath, name, '')) - zip.write(path, path) - log.info("adding '%s'", path) - for name in filenames: - path = os.path.normpath(os.path.join(dirpath, name)) - if os.path.isfile(path): - zip.write(path, path) - log.info("adding '%s'", path) - zip.close() - - return zip_filename - -ARCHIVE_FORMATS = { - 'gztar': (make_tarball, [('compress', 'gzip')], "gzip'ed tar-file"), - 'bztar': (make_tarball, [('compress', 'bzip2')], "bzip2'ed tar-file"), - 'xztar': (make_tarball, [('compress', 'xz')], "xz'ed tar-file"), - 'ztar': (make_tarball, [('compress', 'compress')], "compressed tar file"), - 'tar': (make_tarball, [('compress', None)], "uncompressed tar file"), - 'zip': (make_zipfile, [],"ZIP file") - } - -def check_archive_formats(formats): - """Returns the first format from the 'format' list that is unknown. - - If all formats are known, returns None - """ - for format in formats: - if format not in ARCHIVE_FORMATS: - return format - return None - -def make_archive(base_name, format, root_dir=None, base_dir=None, verbose=0, - dry_run=0, owner=None, group=None): - """Create an archive file (eg. zip or tar). - - 'base_name' is the name of the file to create, minus any format-specific - extension; 'format' is the archive format: one of "zip", "tar", "gztar", - "bztar", "xztar", or "ztar". - - 'root_dir' is a directory that will be the root directory of the - archive; ie. we typically chdir into 'root_dir' before creating the - archive. 'base_dir' is the directory where we start archiving from; - ie. 'base_dir' will be the common prefix of all files and - directories in the archive. 'root_dir' and 'base_dir' both default - to the current directory. Returns the name of the archive file. - - 'owner' and 'group' are used when creating a tar archive. By default, - uses the current owner and group. - """ - save_cwd = os.getcwd() - if root_dir is not None: - log.debug("changing into '%s'", root_dir) - base_name = os.path.abspath(base_name) - if not dry_run: - os.chdir(root_dir) - - if base_dir is None: - base_dir = os.curdir - - kwargs = {'dry_run': dry_run} - - try: - format_info = ARCHIVE_FORMATS[format] - except KeyError: - raise ValueError("unknown archive format '%s'" % format) - - func = format_info[0] - for arg, val in format_info[1]: - kwargs[arg] = val - - if format != 'zip': - kwargs['owner'] = owner - kwargs['group'] = group - - try: - filename = func(base_name, base_dir, **kwargs) - finally: - if root_dir is not None: - log.debug("changing back to '%s'", save_cwd) - os.chdir(save_cwd) - - return filename diff --git a/Lib/distutils/bcppcompiler.py b/Lib/distutils/bcppcompiler.py deleted file mode 100644 index 9f4c432d90e..00000000000 --- a/Lib/distutils/bcppcompiler.py +++ /dev/null @@ -1,393 +0,0 @@ -"""distutils.bcppcompiler - -Contains BorlandCCompiler, an implementation of the abstract CCompiler class -for the Borland C++ compiler. -""" - -# This implementation by Lyle Johnson, based on the original msvccompiler.py -# module and using the directions originally published by Gordon Williams. - -# XXX looks like there's a LOT of overlap between these two classes: -# someone should sit down and factor out the common code as -# WindowsCCompiler! --GPW - - -import os -from distutils.errors import \ - DistutilsExecError, DistutilsPlatformError, \ - CompileError, LibError, LinkError, UnknownFileError -from distutils.ccompiler import \ - CCompiler, gen_preprocess_options, gen_lib_options -from distutils.file_util import write_file -from distutils.dep_util import newer -from distutils import log - -class BCPPCompiler(CCompiler) : - """Concrete class that implements an interface to the Borland C/C++ - compiler, as defined by the CCompiler abstract class. - """ - - compiler_type = 'bcpp' - - # Just set this so CCompiler's constructor doesn't barf. We currently - # don't use the 'set_executables()' bureaucracy provided by CCompiler, - # as it really isn't necessary for this sort of single-compiler class. - # Would be nice to have a consistent interface with UnixCCompiler, - # though, so it's worth thinking about. - executables = {} - - # Private class data (need to distinguish C from C++ source for compiler) - _c_extensions = ['.c'] - _cpp_extensions = ['.cc', '.cpp', '.cxx'] - - # Needed for the filename generation methods provided by the - # base class, CCompiler. - src_extensions = _c_extensions + _cpp_extensions - obj_extension = '.obj' - static_lib_extension = '.lib' - shared_lib_extension = '.dll' - static_lib_format = shared_lib_format = '%s%s' - exe_extension = '.exe' - - - def __init__ (self, - verbose=0, - dry_run=0, - force=0): - - CCompiler.__init__ (self, verbose, dry_run, force) - - # These executables are assumed to all be in the path. - # Borland doesn't seem to use any special registry settings to - # indicate their installation locations. - - self.cc = "bcc32.exe" - self.linker = "ilink32.exe" - self.lib = "tlib.exe" - - self.preprocess_options = None - self.compile_options = ['/tWM', '/O2', '/q', '/g0'] - self.compile_options_debug = ['/tWM', '/Od', '/q', '/g0'] - - self.ldflags_shared = ['/Tpd', '/Gn', '/q', '/x'] - self.ldflags_shared_debug = ['/Tpd', '/Gn', '/q', '/x'] - self.ldflags_static = [] - self.ldflags_exe = ['/Gn', '/q', '/x'] - self.ldflags_exe_debug = ['/Gn', '/q', '/x','/r'] - - - # -- Worker methods ------------------------------------------------ - - def compile(self, sources, - output_dir=None, macros=None, include_dirs=None, debug=0, - extra_preargs=None, extra_postargs=None, depends=None): - - macros, objects, extra_postargs, pp_opts, build = \ - self._setup_compile(output_dir, macros, include_dirs, sources, - depends, extra_postargs) - compile_opts = extra_preargs or [] - compile_opts.append ('-c') - if debug: - compile_opts.extend (self.compile_options_debug) - else: - compile_opts.extend (self.compile_options) - - for obj in objects: - try: - src, ext = build[obj] - except KeyError: - continue - # XXX why do the normpath here? - src = os.path.normpath(src) - obj = os.path.normpath(obj) - # XXX _setup_compile() did a mkpath() too but before the normpath. - # Is it possible to skip the normpath? - self.mkpath(os.path.dirname(obj)) - - if ext == '.res': - # This is already a binary file -- skip it. - continue # the 'for' loop - if ext == '.rc': - # This needs to be compiled to a .res file -- do it now. - try: - self.spawn (["brcc32", "-fo", obj, src]) - except DistutilsExecError as msg: - raise CompileError(msg) - continue # the 'for' loop - - # The next two are both for the real compiler. - if ext in self._c_extensions: - input_opt = "" - elif ext in self._cpp_extensions: - input_opt = "-P" - else: - # Unknown file type -- no extra options. The compiler - # will probably fail, but let it just in case this is a - # file the compiler recognizes even if we don't. - input_opt = "" - - output_opt = "-o" + obj - - # Compiler command line syntax is: "bcc32 [options] file(s)". - # Note that the source file names must appear at the end of - # the command line. - try: - self.spawn ([self.cc] + compile_opts + pp_opts + - [input_opt, output_opt] + - extra_postargs + [src]) - except DistutilsExecError as msg: - raise CompileError(msg) - - return objects - - # compile () - - - def create_static_lib (self, - objects, - output_libname, - output_dir=None, - debug=0, - target_lang=None): - - (objects, output_dir) = self._fix_object_args (objects, output_dir) - output_filename = \ - self.library_filename (output_libname, output_dir=output_dir) - - if self._need_link (objects, output_filename): - lib_args = [output_filename, '/u'] + objects - if debug: - pass # XXX what goes here? - try: - self.spawn ([self.lib] + lib_args) - except DistutilsExecError as msg: - raise LibError(msg) - else: - log.debug("skipping %s (up-to-date)", output_filename) - - # create_static_lib () - - - def link (self, - target_desc, - objects, - output_filename, - output_dir=None, - libraries=None, - library_dirs=None, - runtime_library_dirs=None, - export_symbols=None, - debug=0, - extra_preargs=None, - extra_postargs=None, - build_temp=None, - target_lang=None): - - # XXX this ignores 'build_temp'! should follow the lead of - # msvccompiler.py - - (objects, output_dir) = self._fix_object_args (objects, output_dir) - (libraries, library_dirs, runtime_library_dirs) = \ - self._fix_lib_args (libraries, library_dirs, runtime_library_dirs) - - if runtime_library_dirs: - log.warn("I don't know what to do with 'runtime_library_dirs': %s", - str(runtime_library_dirs)) - - if output_dir is not None: - output_filename = os.path.join (output_dir, output_filename) - - if self._need_link (objects, output_filename): - - # Figure out linker args based on type of target. - if target_desc == CCompiler.EXECUTABLE: - startup_obj = 'c0w32' - if debug: - ld_args = self.ldflags_exe_debug[:] - else: - ld_args = self.ldflags_exe[:] - else: - startup_obj = 'c0d32' - if debug: - ld_args = self.ldflags_shared_debug[:] - else: - ld_args = self.ldflags_shared[:] - - - # Create a temporary exports file for use by the linker - if export_symbols is None: - def_file = '' - else: - head, tail = os.path.split (output_filename) - modname, ext = os.path.splitext (tail) - temp_dir = os.path.dirname(objects[0]) # preserve tree structure - def_file = os.path.join (temp_dir, '%s.def' % modname) - contents = ['EXPORTS'] - for sym in (export_symbols or []): - contents.append(' %s=_%s' % (sym, sym)) - self.execute(write_file, (def_file, contents), - "writing %s" % def_file) - - # Borland C++ has problems with '/' in paths - objects2 = map(os.path.normpath, objects) - # split objects in .obj and .res files - # Borland C++ needs them at different positions in the command line - objects = [startup_obj] - resources = [] - for file in objects2: - (base, ext) = os.path.splitext(os.path.normcase(file)) - if ext == '.res': - resources.append(file) - else: - objects.append(file) - - - for l in library_dirs: - ld_args.append("/L%s" % os.path.normpath(l)) - ld_args.append("/L.") # we sometimes use relative paths - - # list of object files - ld_args.extend(objects) - - # XXX the command-line syntax for Borland C++ is a bit wonky; - # certain filenames are jammed together in one big string, but - # comma-delimited. This doesn't mesh too well with the - # Unix-centric attitude (with a DOS/Windows quoting hack) of - # 'spawn()', so constructing the argument list is a bit - # awkward. Note that doing the obvious thing and jamming all - # the filenames and commas into one argument would be wrong, - # because 'spawn()' would quote any filenames with spaces in - # them. Arghghh!. Apparently it works fine as coded... - - # name of dll/exe file - ld_args.extend([',',output_filename]) - # no map file and start libraries - ld_args.append(',,') - - for lib in libraries: - # see if we find it and if there is a bcpp specific lib - # (xxx_bcpp.lib) - libfile = self.find_library_file(library_dirs, lib, debug) - if libfile is None: - ld_args.append(lib) - # probably a BCPP internal library -- don't warn - else: - # full name which prefers bcpp_xxx.lib over xxx.lib - ld_args.append(libfile) - - # some default libraries - ld_args.append ('import32') - ld_args.append ('cw32mt') - - # def file for export symbols - ld_args.extend([',',def_file]) - # add resource files - ld_args.append(',') - ld_args.extend(resources) - - - if extra_preargs: - ld_args[:0] = extra_preargs - if extra_postargs: - ld_args.extend(extra_postargs) - - self.mkpath (os.path.dirname (output_filename)) - try: - self.spawn ([self.linker] + ld_args) - except DistutilsExecError as msg: - raise LinkError(msg) - - else: - log.debug("skipping %s (up-to-date)", output_filename) - - # link () - - # -- Miscellaneous methods ----------------------------------------- - - - def find_library_file (self, dirs, lib, debug=0): - # List of effective library names to try, in order of preference: - # xxx_bcpp.lib is better than xxx.lib - # and xxx_d.lib is better than xxx.lib if debug is set - # - # The "_bcpp" suffix is to handle a Python installation for people - # with multiple compilers (primarily Distutils hackers, I suspect - # ;-). The idea is they'd have one static library for each - # compiler they care about, since (almost?) every Windows compiler - # seems to have a different format for static libraries. - if debug: - dlib = (lib + "_d") - try_names = (dlib + "_bcpp", lib + "_bcpp", dlib, lib) - else: - try_names = (lib + "_bcpp", lib) - - for dir in dirs: - for name in try_names: - libfile = os.path.join(dir, self.library_filename(name)) - if os.path.exists(libfile): - return libfile - else: - # Oops, didn't find it in *any* of 'dirs' - return None - - # overwrite the one from CCompiler to support rc and res-files - def object_filenames (self, - source_filenames, - strip_dir=0, - output_dir=''): - if output_dir is None: output_dir = '' - obj_names = [] - for src_name in source_filenames: - # use normcase to make sure '.rc' is really '.rc' and not '.RC' - (base, ext) = os.path.splitext (os.path.normcase(src_name)) - if ext not in (self.src_extensions + ['.rc','.res']): - raise UnknownFileError("unknown file type '%s' (from '%s')" % \ - (ext, src_name)) - if strip_dir: - base = os.path.basename (base) - if ext == '.res': - # these can go unchanged - obj_names.append (os.path.join (output_dir, base + ext)) - elif ext == '.rc': - # these need to be compiled to .res-files - obj_names.append (os.path.join (output_dir, base + '.res')) - else: - obj_names.append (os.path.join (output_dir, - base + self.obj_extension)) - return obj_names - - # object_filenames () - - def preprocess (self, - source, - output_file=None, - macros=None, - include_dirs=None, - extra_preargs=None, - extra_postargs=None): - - (_, macros, include_dirs) = \ - self._fix_compile_args(None, macros, include_dirs) - pp_opts = gen_preprocess_options(macros, include_dirs) - pp_args = ['cpp32.exe'] + pp_opts - if output_file is not None: - pp_args.append('-o' + output_file) - if extra_preargs: - pp_args[:0] = extra_preargs - if extra_postargs: - pp_args.extend(extra_postargs) - pp_args.append(source) - - # We need to preprocess: either we're being forced to, or the - # source file is newer than the target (or the target doesn't - # exist). - if self.force or output_file is None or newer(source, output_file): - if output_file: - self.mkpath(os.path.dirname(output_file)) - try: - self.spawn(pp_args) - except DistutilsExecError as msg: - print(msg) - raise CompileError(msg) - - # preprocess() diff --git a/Lib/distutils/ccompiler.py b/Lib/distutils/ccompiler.py deleted file mode 100644 index b71d1d39bcd..00000000000 --- a/Lib/distutils/ccompiler.py +++ /dev/null @@ -1,1115 +0,0 @@ -"""distutils.ccompiler - -Contains CCompiler, an abstract base class that defines the interface -for the Distutils compiler abstraction model.""" - -import sys, os, re -from distutils.errors import * -from distutils.spawn import spawn -from distutils.file_util import move_file -from distutils.dir_util import mkpath -from distutils.dep_util import newer_pairwise, newer_group -from distutils.util import split_quoted, execute -from distutils import log - -class CCompiler: - """Abstract base class to define the interface that must be implemented - by real compiler classes. Also has some utility methods used by - several compiler classes. - - The basic idea behind a compiler abstraction class is that each - instance can be used for all the compile/link steps in building a - single project. Thus, attributes common to all of those compile and - link steps -- include directories, macros to define, libraries to link - against, etc. -- are attributes of the compiler instance. To allow for - variability in how individual files are treated, most of those - attributes may be varied on a per-compilation or per-link basis. - """ - - # 'compiler_type' is a class attribute that identifies this class. It - # keeps code that wants to know what kind of compiler it's dealing with - # from having to import all possible compiler classes just to do an - # 'isinstance'. In concrete CCompiler subclasses, 'compiler_type' - # should really, really be one of the keys of the 'compiler_class' - # dictionary (see below -- used by the 'new_compiler()' factory - # function) -- authors of new compiler interface classes are - # responsible for updating 'compiler_class'! - compiler_type = None - - # XXX things not handled by this compiler abstraction model: - # * client can't provide additional options for a compiler, - # e.g. warning, optimization, debugging flags. Perhaps this - # should be the domain of concrete compiler abstraction classes - # (UnixCCompiler, MSVCCompiler, etc.) -- or perhaps the base - # class should have methods for the common ones. - # * can't completely override the include or library searchg - # path, ie. no "cc -I -Idir1 -Idir2" or "cc -L -Ldir1 -Ldir2". - # I'm not sure how widely supported this is even by Unix - # compilers, much less on other platforms. And I'm even less - # sure how useful it is; maybe for cross-compiling, but - # support for that is a ways off. (And anyways, cross - # compilers probably have a dedicated binary with the - # right paths compiled in. I hope.) - # * can't do really freaky things with the library list/library - # dirs, e.g. "-Ldir1 -lfoo -Ldir2 -lfoo" to link against - # different versions of libfoo.a in different locations. I - # think this is useless without the ability to null out the - # library search path anyways. - - - # Subclasses that rely on the standard filename generation methods - # implemented below should override these; see the comment near - # those methods ('object_filenames()' et. al.) for details: - src_extensions = None # list of strings - obj_extension = None # string - static_lib_extension = None - shared_lib_extension = None # string - static_lib_format = None # format string - shared_lib_format = None # prob. same as static_lib_format - exe_extension = None # string - - # Default language settings. language_map is used to detect a source - # file or Extension target language, checking source filenames. - # language_order is used to detect the language precedence, when deciding - # what language to use when mixing source types. For example, if some - # extension has two files with ".c" extension, and one with ".cpp", it - # is still linked as c++. - language_map = {".c" : "c", - ".cc" : "c++", - ".cpp" : "c++", - ".cxx" : "c++", - ".m" : "objc", - } - language_order = ["c++", "objc", "c"] - - def __init__(self, verbose=0, dry_run=0, force=0): - self.dry_run = dry_run - self.force = force - self.verbose = verbose - - # 'output_dir': a common output directory for object, library, - # shared object, and shared library files - self.output_dir = None - - # 'macros': a list of macro definitions (or undefinitions). A - # macro definition is a 2-tuple (name, value), where the value is - # either a string or None (no explicit value). A macro - # undefinition is a 1-tuple (name,). - self.macros = [] - - # 'include_dirs': a list of directories to search for include files - self.include_dirs = [] - - # 'libraries': a list of libraries to include in any link - # (library names, not filenames: eg. "foo" not "libfoo.a") - self.libraries = [] - - # 'library_dirs': a list of directories to search for libraries - self.library_dirs = [] - - # 'runtime_library_dirs': a list of directories to search for - # shared libraries/objects at runtime - self.runtime_library_dirs = [] - - # 'objects': a list of object files (or similar, such as explicitly - # named library files) to include on any link - self.objects = [] - - for key in self.executables.keys(): - self.set_executable(key, self.executables[key]) - - def set_executables(self, **kwargs): - """Define the executables (and options for them) that will be run - to perform the various stages of compilation. The exact set of - executables that may be specified here depends on the compiler - class (via the 'executables' class attribute), but most will have: - compiler the C/C++ compiler - linker_so linker used to create shared objects and libraries - linker_exe linker used to create binary executables - archiver static library creator - - On platforms with a command-line (Unix, DOS/Windows), each of these - is a string that will be split into executable name and (optional) - list of arguments. (Splitting the string is done similarly to how - Unix shells operate: words are delimited by spaces, but quotes and - backslashes can override this. See - 'distutils.util.split_quoted()'.) - """ - - # Note that some CCompiler implementation classes will define class - # attributes 'cpp', 'cc', etc. with hard-coded executable names; - # this is appropriate when a compiler class is for exactly one - # compiler/OS combination (eg. MSVCCompiler). Other compiler - # classes (UnixCCompiler, in particular) are driven by information - # discovered at run-time, since there are many different ways to do - # basically the same things with Unix C compilers. - - for key in kwargs: - if key not in self.executables: - raise ValueError("unknown executable '%s' for class %s" % - (key, self.__class__.__name__)) - self.set_executable(key, kwargs[key]) - - def set_executable(self, key, value): - if isinstance(value, str): - setattr(self, key, split_quoted(value)) - else: - setattr(self, key, value) - - def _find_macro(self, name): - i = 0 - for defn in self.macros: - if defn[0] == name: - return i - i += 1 - return None - - def _check_macro_definitions(self, definitions): - """Ensures that every element of 'definitions' is a valid macro - definition, ie. either (name,value) 2-tuple or a (name,) tuple. Do - nothing if all definitions are OK, raise TypeError otherwise. - """ - for defn in definitions: - if not (isinstance(defn, tuple) and - (len(defn) in (1, 2) and - (isinstance (defn[1], str) or defn[1] is None)) and - isinstance (defn[0], str)): - raise TypeError(("invalid macro definition '%s': " % defn) + \ - "must be tuple (string,), (string, string), or " + \ - "(string, None)") - - - # -- Bookkeeping methods ------------------------------------------- - - def define_macro(self, name, value=None): - """Define a preprocessor macro for all compilations driven by this - compiler object. The optional parameter 'value' should be a - string; if it is not supplied, then the macro will be defined - without an explicit value and the exact outcome depends on the - compiler used (XXX true? does ANSI say anything about this?) - """ - # Delete from the list of macro definitions/undefinitions if - # already there (so that this one will take precedence). - i = self._find_macro (name) - if i is not None: - del self.macros[i] - - self.macros.append((name, value)) - - def undefine_macro(self, name): - """Undefine a preprocessor macro for all compilations driven by - this compiler object. If the same macro is defined by - 'define_macro()' and undefined by 'undefine_macro()' the last call - takes precedence (including multiple redefinitions or - undefinitions). If the macro is redefined/undefined on a - per-compilation basis (ie. in the call to 'compile()'), then that - takes precedence. - """ - # Delete from the list of macro definitions/undefinitions if - # already there (so that this one will take precedence). - i = self._find_macro (name) - if i is not None: - del self.macros[i] - - undefn = (name,) - self.macros.append(undefn) - - def add_include_dir(self, dir): - """Add 'dir' to the list of directories that will be searched for - header files. The compiler is instructed to search directories in - the order in which they are supplied by successive calls to - 'add_include_dir()'. - """ - self.include_dirs.append(dir) - - def set_include_dirs(self, dirs): - """Set the list of directories that will be searched to 'dirs' (a - list of strings). Overrides any preceding calls to - 'add_include_dir()'; subsequence calls to 'add_include_dir()' add - to the list passed to 'set_include_dirs()'. This does not affect - any list of standard include directories that the compiler may - search by default. - """ - self.include_dirs = dirs[:] - - def add_library(self, libname): - """Add 'libname' to the list of libraries that will be included in - all links driven by this compiler object. Note that 'libname' - should *not* be the name of a file containing a library, but the - name of the library itself: the actual filename will be inferred by - the linker, the compiler, or the compiler class (depending on the - platform). - - The linker will be instructed to link against libraries in the - order they were supplied to 'add_library()' and/or - 'set_libraries()'. It is perfectly valid to duplicate library - names; the linker will be instructed to link against libraries as - many times as they are mentioned. - """ - self.libraries.append(libname) - - def set_libraries(self, libnames): - """Set the list of libraries to be included in all links driven by - this compiler object to 'libnames' (a list of strings). This does - not affect any standard system libraries that the linker may - include by default. - """ - self.libraries = libnames[:] - - def add_library_dir(self, dir): - """Add 'dir' to the list of directories that will be searched for - libraries specified to 'add_library()' and 'set_libraries()'. The - linker will be instructed to search for libraries in the order they - are supplied to 'add_library_dir()' and/or 'set_library_dirs()'. - """ - self.library_dirs.append(dir) - - def set_library_dirs(self, dirs): - """Set the list of library search directories to 'dirs' (a list of - strings). This does not affect any standard library search path - that the linker may search by default. - """ - self.library_dirs = dirs[:] - - def add_runtime_library_dir(self, dir): - """Add 'dir' to the list of directories that will be searched for - shared libraries at runtime. - """ - self.runtime_library_dirs.append(dir) - - def set_runtime_library_dirs(self, dirs): - """Set the list of directories to search for shared libraries at - runtime to 'dirs' (a list of strings). This does not affect any - standard search path that the runtime linker may search by - default. - """ - self.runtime_library_dirs = dirs[:] - - def add_link_object(self, object): - """Add 'object' to the list of object files (or analogues, such as - explicitly named library files or the output of "resource - compilers") to be included in every link driven by this compiler - object. - """ - self.objects.append(object) - - def set_link_objects(self, objects): - """Set the list of object files (or analogues) to be included in - every link to 'objects'. This does not affect any standard object - files that the linker may include by default (such as system - libraries). - """ - self.objects = objects[:] - - - # -- Private utility methods -------------------------------------- - # (here for the convenience of subclasses) - - # Helper method to prep compiler in subclass compile() methods - - def _setup_compile(self, outdir, macros, incdirs, sources, depends, - extra): - """Process arguments and decide which source files to compile.""" - if outdir is None: - outdir = self.output_dir - elif not isinstance(outdir, str): - raise TypeError("'output_dir' must be a string or None") - - if macros is None: - macros = self.macros - elif isinstance(macros, list): - macros = macros + (self.macros or []) - else: - raise TypeError("'macros' (if supplied) must be a list of tuples") - - if incdirs is None: - incdirs = self.include_dirs - elif isinstance(incdirs, (list, tuple)): - incdirs = list(incdirs) + (self.include_dirs or []) - else: - raise TypeError( - "'include_dirs' (if supplied) must be a list of strings") - - if extra is None: - extra = [] - - # Get the list of expected output (object) files - objects = self.object_filenames(sources, strip_dir=0, - output_dir=outdir) - assert len(objects) == len(sources) - - pp_opts = gen_preprocess_options(macros, incdirs) - - build = {} - for i in range(len(sources)): - src = sources[i] - obj = objects[i] - ext = os.path.splitext(src)[1] - self.mkpath(os.path.dirname(obj)) - build[obj] = (src, ext) - - return macros, objects, extra, pp_opts, build - - def _get_cc_args(self, pp_opts, debug, before): - # works for unixccompiler, cygwinccompiler - cc_args = pp_opts + ['-c'] - if debug: - cc_args[:0] = ['-g'] - if before: - cc_args[:0] = before - return cc_args - - def _fix_compile_args(self, output_dir, macros, include_dirs): - """Typecheck and fix-up some of the arguments to the 'compile()' - method, and return fixed-up values. Specifically: if 'output_dir' - is None, replaces it with 'self.output_dir'; ensures that 'macros' - is a list, and augments it with 'self.macros'; ensures that - 'include_dirs' is a list, and augments it with 'self.include_dirs'. - Guarantees that the returned values are of the correct type, - i.e. for 'output_dir' either string or None, and for 'macros' and - 'include_dirs' either list or None. - """ - if output_dir is None: - output_dir = self.output_dir - elif not isinstance(output_dir, str): - raise TypeError("'output_dir' must be a string or None") - - if macros is None: - macros = self.macros - elif isinstance(macros, list): - macros = macros + (self.macros or []) - else: - raise TypeError("'macros' (if supplied) must be a list of tuples") - - if include_dirs is None: - include_dirs = self.include_dirs - elif isinstance(include_dirs, (list, tuple)): - include_dirs = list(include_dirs) + (self.include_dirs or []) - else: - raise TypeError( - "'include_dirs' (if supplied) must be a list of strings") - - return output_dir, macros, include_dirs - - def _prep_compile(self, sources, output_dir, depends=None): - """Decide which souce files must be recompiled. - - Determine the list of object files corresponding to 'sources', - and figure out which ones really need to be recompiled. - Return a list of all object files and a dictionary telling - which source files can be skipped. - """ - # Get the list of expected output (object) files - objects = self.object_filenames(sources, output_dir=output_dir) - assert len(objects) == len(sources) - - # Return an empty dict for the "which source files can be skipped" - # return value to preserve API compatibility. - return objects, {} - - def _fix_object_args(self, objects, output_dir): - """Typecheck and fix up some arguments supplied to various methods. - Specifically: ensure that 'objects' is a list; if output_dir is - None, replace with self.output_dir. Return fixed versions of - 'objects' and 'output_dir'. - """ - if not isinstance(objects, (list, tuple)): - raise TypeError("'objects' must be a list or tuple of strings") - objects = list(objects) - - if output_dir is None: - output_dir = self.output_dir - elif not isinstance(output_dir, str): - raise TypeError("'output_dir' must be a string or None") - - return (objects, output_dir) - - def _fix_lib_args(self, libraries, library_dirs, runtime_library_dirs): - """Typecheck and fix up some of the arguments supplied to the - 'link_*' methods. Specifically: ensure that all arguments are - lists, and augment them with their permanent versions - (eg. 'self.libraries' augments 'libraries'). Return a tuple with - fixed versions of all arguments. - """ - if libraries is None: - libraries = self.libraries - elif isinstance(libraries, (list, tuple)): - libraries = list (libraries) + (self.libraries or []) - else: - raise TypeError( - "'libraries' (if supplied) must be a list of strings") - - if library_dirs is None: - library_dirs = self.library_dirs - elif isinstance(library_dirs, (list, tuple)): - library_dirs = list (library_dirs) + (self.library_dirs or []) - else: - raise TypeError( - "'library_dirs' (if supplied) must be a list of strings") - - if runtime_library_dirs is None: - runtime_library_dirs = self.runtime_library_dirs - elif isinstance(runtime_library_dirs, (list, tuple)): - runtime_library_dirs = (list(runtime_library_dirs) + - (self.runtime_library_dirs or [])) - else: - raise TypeError("'runtime_library_dirs' (if supplied) " - "must be a list of strings") - - return (libraries, library_dirs, runtime_library_dirs) - - def _need_link(self, objects, output_file): - """Return true if we need to relink the files listed in 'objects' - to recreate 'output_file'. - """ - if self.force: - return True - else: - if self.dry_run: - newer = newer_group (objects, output_file, missing='newer') - else: - newer = newer_group (objects, output_file) - return newer - - def detect_language(self, sources): - """Detect the language of a given file, or list of files. Uses - language_map, and language_order to do the job. - """ - if not isinstance(sources, list): - sources = [sources] - lang = None - index = len(self.language_order) - for source in sources: - base, ext = os.path.splitext(source) - extlang = self.language_map.get(ext) - try: - extindex = self.language_order.index(extlang) - if extindex < index: - lang = extlang - index = extindex - except ValueError: - pass - return lang - - - # -- Worker methods ------------------------------------------------ - # (must be implemented by subclasses) - - def preprocess(self, source, output_file=None, macros=None, - include_dirs=None, extra_preargs=None, extra_postargs=None): - """Preprocess a single C/C++ source file, named in 'source'. - Output will be written to file named 'output_file', or stdout if - 'output_file' not supplied. 'macros' is a list of macro - definitions as for 'compile()', which will augment the macros set - with 'define_macro()' and 'undefine_macro()'. 'include_dirs' is a - list of directory names that will be added to the default list. - - Raises PreprocessError on failure. - """ - pass - - def compile(self, sources, output_dir=None, macros=None, - include_dirs=None, debug=0, extra_preargs=None, - extra_postargs=None, depends=None): - """Compile one or more source files. - - 'sources' must be a list of filenames, most likely C/C++ - files, but in reality anything that can be handled by a - particular compiler and compiler class (eg. MSVCCompiler can - handle resource files in 'sources'). Return a list of object - filenames, one per source filename in 'sources'. Depending on - the implementation, not all source files will necessarily be - compiled, but all corresponding object filenames will be - returned. - - If 'output_dir' is given, object files will be put under it, while - retaining their original path component. That is, "foo/bar.c" - normally compiles to "foo/bar.o" (for a Unix implementation); if - 'output_dir' is "build", then it would compile to - "build/foo/bar.o". - - 'macros', if given, must be a list of macro definitions. A macro - definition is either a (name, value) 2-tuple or a (name,) 1-tuple. - The former defines a macro; if the value is None, the macro is - defined without an explicit value. The 1-tuple case undefines a - macro. Later definitions/redefinitions/ undefinitions take - precedence. - - 'include_dirs', if given, must be a list of strings, the - directories to add to the default include file search path for this - compilation only. - - 'debug' is a boolean; if true, the compiler will be instructed to - output debug symbols in (or alongside) the object file(s). - - 'extra_preargs' and 'extra_postargs' are implementation- dependent. - On platforms that have the notion of a command-line (e.g. Unix, - DOS/Windows), they are most likely lists of strings: extra - command-line arguments to prepand/append to the compiler command - line. On other platforms, consult the implementation class - documentation. In any event, they are intended as an escape hatch - for those occasions when the abstract compiler framework doesn't - cut the mustard. - - 'depends', if given, is a list of filenames that all targets - depend on. If a source file is older than any file in - depends, then the source file will be recompiled. This - supports dependency tracking, but only at a coarse - granularity. - - Raises CompileError on failure. - """ - # A concrete compiler class can either override this method - # entirely or implement _compile(). - macros, objects, extra_postargs, pp_opts, build = \ - self._setup_compile(output_dir, macros, include_dirs, sources, - depends, extra_postargs) - cc_args = self._get_cc_args(pp_opts, debug, extra_preargs) - - for obj in objects: - try: - src, ext = build[obj] - except KeyError: - continue - self._compile(obj, src, ext, cc_args, extra_postargs, pp_opts) - - # Return *all* object filenames, not just the ones we just built. - return objects - - def _compile(self, obj, src, ext, cc_args, extra_postargs, pp_opts): - """Compile 'src' to product 'obj'.""" - # A concrete compiler class that does not override compile() - # should implement _compile(). - pass - - def create_static_lib(self, objects, output_libname, output_dir=None, - debug=0, target_lang=None): - """Link a bunch of stuff together to create a static library file. - The "bunch of stuff" consists of the list of object files supplied - as 'objects', the extra object files supplied to - 'add_link_object()' and/or 'set_link_objects()', the libraries - supplied to 'add_library()' and/or 'set_libraries()', and the - libraries supplied as 'libraries' (if any). - - 'output_libname' should be a library name, not a filename; the - filename will be inferred from the library name. 'output_dir' is - the directory where the library file will be put. - - 'debug' is a boolean; if true, debugging information will be - included in the library (note that on most platforms, it is the - compile step where this matters: the 'debug' flag is included here - just for consistency). - - 'target_lang' is the target language for which the given objects - are being compiled. This allows specific linkage time treatment of - certain languages. - - Raises LibError on failure. - """ - pass - - - # values for target_desc parameter in link() - SHARED_OBJECT = "shared_object" - SHARED_LIBRARY = "shared_library" - EXECUTABLE = "executable" - - def link(self, - target_desc, - objects, - output_filename, - output_dir=None, - libraries=None, - library_dirs=None, - runtime_library_dirs=None, - export_symbols=None, - debug=0, - extra_preargs=None, - extra_postargs=None, - build_temp=None, - target_lang=None): - """Link a bunch of stuff together to create an executable or - shared library file. - - The "bunch of stuff" consists of the list of object files supplied - as 'objects'. 'output_filename' should be a filename. If - 'output_dir' is supplied, 'output_filename' is relative to it - (i.e. 'output_filename' can provide directory components if - needed). - - 'libraries' is a list of libraries to link against. These are - library names, not filenames, since they're translated into - filenames in a platform-specific way (eg. "foo" becomes "libfoo.a" - on Unix and "foo.lib" on DOS/Windows). However, they can include a - directory component, which means the linker will look in that - specific directory rather than searching all the normal locations. - - 'library_dirs', if supplied, should be a list of directories to - search for libraries that were specified as bare library names - (ie. no directory component). These are on top of the system - default and those supplied to 'add_library_dir()' and/or - 'set_library_dirs()'. 'runtime_library_dirs' is a list of - directories that will be embedded into the shared library and used - to search for other shared libraries that *it* depends on at - run-time. (This may only be relevant on Unix.) - - 'export_symbols' is a list of symbols that the shared library will - export. (This appears to be relevant only on Windows.) - - 'debug' is as for 'compile()' and 'create_static_lib()', with the - slight distinction that it actually matters on most platforms (as - opposed to 'create_static_lib()', which includes a 'debug' flag - mostly for form's sake). - - 'extra_preargs' and 'extra_postargs' are as for 'compile()' (except - of course that they supply command-line arguments for the - particular linker being used). - - 'target_lang' is the target language for which the given objects - are being compiled. This allows specific linkage time treatment of - certain languages. - - Raises LinkError on failure. - """ - raise NotImplementedError - - - # Old 'link_*()' methods, rewritten to use the new 'link()' method. - - def link_shared_lib(self, - objects, - output_libname, - output_dir=None, - libraries=None, - library_dirs=None, - runtime_library_dirs=None, - export_symbols=None, - debug=0, - extra_preargs=None, - extra_postargs=None, - build_temp=None, - target_lang=None): - self.link(CCompiler.SHARED_LIBRARY, objects, - self.library_filename(output_libname, lib_type='shared'), - output_dir, - libraries, library_dirs, runtime_library_dirs, - export_symbols, debug, - extra_preargs, extra_postargs, build_temp, target_lang) - - - def link_shared_object(self, - objects, - output_filename, - output_dir=None, - libraries=None, - library_dirs=None, - runtime_library_dirs=None, - export_symbols=None, - debug=0, - extra_preargs=None, - extra_postargs=None, - build_temp=None, - target_lang=None): - self.link(CCompiler.SHARED_OBJECT, objects, - output_filename, output_dir, - libraries, library_dirs, runtime_library_dirs, - export_symbols, debug, - extra_preargs, extra_postargs, build_temp, target_lang) - - - def link_executable(self, - objects, - output_progname, - output_dir=None, - libraries=None, - library_dirs=None, - runtime_library_dirs=None, - debug=0, - extra_preargs=None, - extra_postargs=None, - target_lang=None): - self.link(CCompiler.EXECUTABLE, objects, - self.executable_filename(output_progname), output_dir, - libraries, library_dirs, runtime_library_dirs, None, - debug, extra_preargs, extra_postargs, None, target_lang) - - - # -- Miscellaneous methods ----------------------------------------- - # These are all used by the 'gen_lib_options() function; there is - # no appropriate default implementation so subclasses should - # implement all of these. - - def library_dir_option(self, dir): - """Return the compiler option to add 'dir' to the list of - directories searched for libraries. - """ - raise NotImplementedError - - def runtime_library_dir_option(self, dir): - """Return the compiler option to add 'dir' to the list of - directories searched for runtime libraries. - """ - raise NotImplementedError - - def library_option(self, lib): - """Return the compiler option to add 'lib' to the list of libraries - linked into the shared library or executable. - """ - raise NotImplementedError - - def has_function(self, funcname, includes=None, include_dirs=None, - libraries=None, library_dirs=None): - """Return a boolean indicating whether funcname is supported on - the current platform. The optional arguments can be used to - augment the compilation environment. - """ - # this can't be included at module scope because it tries to - # import math which might not be available at that point - maybe - # the necessary logic should just be inlined? - import tempfile - if includes is None: - includes = [] - if include_dirs is None: - include_dirs = [] - if libraries is None: - libraries = [] - if library_dirs is None: - library_dirs = [] - fd, fname = tempfile.mkstemp(".c", funcname, text=True) - f = os.fdopen(fd, "w") - try: - for incl in includes: - f.write("""#include "%s"\n""" % incl) - f.write("""\ -main (int argc, char **argv) { - %s(); -} -""" % funcname) - finally: - f.close() - try: - objects = self.compile([fname], include_dirs=include_dirs) - except CompileError: - return False - - try: - self.link_executable(objects, "a.out", - libraries=libraries, - library_dirs=library_dirs) - except (LinkError, TypeError): - return False - return True - - def find_library_file (self, dirs, lib, debug=0): - """Search the specified list of directories for a static or shared - library file 'lib' and return the full path to that file. If - 'debug' true, look for a debugging version (if that makes sense on - the current platform). Return None if 'lib' wasn't found in any of - the specified directories. - """ - raise NotImplementedError - - # -- Filename generation methods ----------------------------------- - - # The default implementation of the filename generating methods are - # prejudiced towards the Unix/DOS/Windows view of the world: - # * object files are named by replacing the source file extension - # (eg. .c/.cpp -> .o/.obj) - # * library files (shared or static) are named by plugging the - # library name and extension into a format string, eg. - # "lib%s.%s" % (lib_name, ".a") for Unix static libraries - # * executables are named by appending an extension (possibly - # empty) to the program name: eg. progname + ".exe" for - # Windows - # - # To reduce redundant code, these methods expect to find - # several attributes in the current object (presumably defined - # as class attributes): - # * src_extensions - - # list of C/C++ source file extensions, eg. ['.c', '.cpp'] - # * obj_extension - - # object file extension, eg. '.o' or '.obj' - # * static_lib_extension - - # extension for static library files, eg. '.a' or '.lib' - # * shared_lib_extension - - # extension for shared library/object files, eg. '.so', '.dll' - # * static_lib_format - - # format string for generating static library filenames, - # eg. 'lib%s.%s' or '%s.%s' - # * shared_lib_format - # format string for generating shared library filenames - # (probably same as static_lib_format, since the extension - # is one of the intended parameters to the format string) - # * exe_extension - - # extension for executable files, eg. '' or '.exe' - - def object_filenames(self, source_filenames, strip_dir=0, output_dir=''): - if output_dir is None: - output_dir = '' - obj_names = [] - for src_name in source_filenames: - base, ext = os.path.splitext(src_name) - base = os.path.splitdrive(base)[1] # Chop off the drive - base = base[os.path.isabs(base):] # If abs, chop off leading / - if ext not in self.src_extensions: - raise UnknownFileError( - "unknown file type '%s' (from '%s')" % (ext, src_name)) - if strip_dir: - base = os.path.basename(base) - obj_names.append(os.path.join(output_dir, - base + self.obj_extension)) - return obj_names - - def shared_object_filename(self, basename, strip_dir=0, output_dir=''): - assert output_dir is not None - if strip_dir: - basename = os.path.basename(basename) - return os.path.join(output_dir, basename + self.shared_lib_extension) - - def executable_filename(self, basename, strip_dir=0, output_dir=''): - assert output_dir is not None - if strip_dir: - basename = os.path.basename(basename) - return os.path.join(output_dir, basename + (self.exe_extension or '')) - - def library_filename(self, libname, lib_type='static', # or 'shared' - strip_dir=0, output_dir=''): - assert output_dir is not None - if lib_type not in ("static", "shared", "dylib", "xcode_stub"): - raise ValueError( - "'lib_type' must be \"static\", \"shared\", \"dylib\", or \"xcode_stub\"") - fmt = getattr(self, lib_type + "_lib_format") - ext = getattr(self, lib_type + "_lib_extension") - - dir, base = os.path.split(libname) - filename = fmt % (base, ext) - if strip_dir: - dir = '' - - return os.path.join(output_dir, dir, filename) - - - # -- Utility methods ----------------------------------------------- - - def announce(self, msg, level=1): - log.debug(msg) - - def debug_print(self, msg): - from distutils.debug import DEBUG - if DEBUG: - print(msg) - - def warn(self, msg): - sys.stderr.write("warning: %s\n" % msg) - - def execute(self, func, args, msg=None, level=1): - execute(func, args, msg, self.dry_run) - - def spawn(self, cmd): - spawn(cmd, dry_run=self.dry_run) - - def move_file(self, src, dst): - return move_file(src, dst, dry_run=self.dry_run) - - def mkpath (self, name, mode=0o777): - mkpath(name, mode, dry_run=self.dry_run) - - -# Map a sys.platform/os.name ('posix', 'nt') to the default compiler -# type for that platform. Keys are interpreted as re match -# patterns. Order is important; platform mappings are preferred over -# OS names. -_default_compilers = ( - - # Platform string mappings - - # on a cygwin built python we can use gcc like an ordinary UNIXish - # compiler - ('cygwin.*', 'unix'), - - # OS name mappings - ('posix', 'unix'), - ('nt', 'msvc'), - - ) - -def get_default_compiler(osname=None, platform=None): - """Determine the default compiler to use for the given platform. - - osname should be one of the standard Python OS names (i.e. the - ones returned by os.name) and platform the common value - returned by sys.platform for the platform in question. - - The default values are os.name and sys.platform in case the - parameters are not given. - """ - if osname is None: - osname = os.name - if platform is None: - platform = sys.platform - for pattern, compiler in _default_compilers: - if re.match(pattern, platform) is not None or \ - re.match(pattern, osname) is not None: - return compiler - # Default to Unix compiler - return 'unix' - -# Map compiler types to (module_name, class_name) pairs -- ie. where to -# find the code that implements an interface to this compiler. (The module -# is assumed to be in the 'distutils' package.) -compiler_class = { 'unix': ('unixccompiler', 'UnixCCompiler', - "standard UNIX-style compiler"), - 'msvc': ('_msvccompiler', 'MSVCCompiler', - "Microsoft Visual C++"), - 'cygwin': ('cygwinccompiler', 'CygwinCCompiler', - "Cygwin port of GNU C Compiler for Win32"), - 'mingw32': ('cygwinccompiler', 'Mingw32CCompiler', - "Mingw32 port of GNU C Compiler for Win32"), - 'bcpp': ('bcppcompiler', 'BCPPCompiler', - "Borland C++ Compiler"), - } - -def show_compilers(): - """Print list of available compilers (used by the "--help-compiler" - options to "build", "build_ext", "build_clib"). - """ - # XXX this "knows" that the compiler option it's describing is - # "--compiler", which just happens to be the case for the three - # commands that use it. - from distutils.fancy_getopt import FancyGetopt - compilers = [] - for compiler in compiler_class.keys(): - compilers.append(("compiler="+compiler, None, - compiler_class[compiler][2])) - compilers.sort() - pretty_printer = FancyGetopt(compilers) - pretty_printer.print_help("List of available compilers:") - - -def new_compiler(plat=None, compiler=None, verbose=0, dry_run=0, force=0): - """Generate an instance of some CCompiler subclass for the supplied - platform/compiler combination. 'plat' defaults to 'os.name' - (eg. 'posix', 'nt'), and 'compiler' defaults to the default compiler - for that platform. Currently only 'posix' and 'nt' are supported, and - the default compilers are "traditional Unix interface" (UnixCCompiler - class) and Visual C++ (MSVCCompiler class). Note that it's perfectly - possible to ask for a Unix compiler object under Windows, and a - Microsoft compiler object under Unix -- if you supply a value for - 'compiler', 'plat' is ignored. - """ - if plat is None: - plat = os.name - - try: - if compiler is None: - compiler = get_default_compiler(plat) - - (module_name, class_name, long_description) = compiler_class[compiler] - except KeyError: - msg = "don't know how to compile C/C++ code on platform '%s'" % plat - if compiler is not None: - msg = msg + " with '%s' compiler" % compiler - raise DistutilsPlatformError(msg) - - try: - module_name = "distutils." + module_name - __import__ (module_name) - module = sys.modules[module_name] - klass = vars(module)[class_name] - except ImportError: - raise DistutilsModuleError( - "can't compile C/C++ code: unable to load module '%s'" % \ - module_name) - except KeyError: - raise DistutilsModuleError( - "can't compile C/C++ code: unable to find class '%s' " - "in module '%s'" % (class_name, module_name)) - - # XXX The None is necessary to preserve backwards compatibility - # with classes that expect verbose to be the first positional - # argument. - return klass(None, dry_run, force) - - -def gen_preprocess_options(macros, include_dirs): - """Generate C pre-processor options (-D, -U, -I) as used by at least - two types of compilers: the typical Unix compiler and Visual C++. - 'macros' is the usual thing, a list of 1- or 2-tuples, where (name,) - means undefine (-U) macro 'name', and (name,value) means define (-D) - macro 'name' to 'value'. 'include_dirs' is just a list of directory - names to be added to the header file search path (-I). Returns a list - of command-line options suitable for either Unix compilers or Visual - C++. - """ - # XXX it would be nice (mainly aesthetic, and so we don't generate - # stupid-looking command lines) to go over 'macros' and eliminate - # redundant definitions/undefinitions (ie. ensure that only the - # latest mention of a particular macro winds up on the command - # line). I don't think it's essential, though, since most (all?) - # Unix C compilers only pay attention to the latest -D or -U - # mention of a macro on their command line. Similar situation for - # 'include_dirs'. I'm punting on both for now. Anyways, weeding out - # redundancies like this should probably be the province of - # CCompiler, since the data structures used are inherited from it - # and therefore common to all CCompiler classes. - pp_opts = [] - for macro in macros: - if not (isinstance(macro, tuple) and 1 <= len(macro) <= 2): - raise TypeError( - "bad macro definition '%s': " - "each element of 'macros' list must be a 1- or 2-tuple" - % macro) - - if len(macro) == 1: # undefine this macro - pp_opts.append("-U%s" % macro[0]) - elif len(macro) == 2: - if macro[1] is None: # define with no explicit value - pp_opts.append("-D%s" % macro[0]) - else: - # XXX *don't* need to be clever about quoting the - # macro value here, because we're going to avoid the - # shell at all costs when we spawn the command! - pp_opts.append("-D%s=%s" % macro) - - for dir in include_dirs: - pp_opts.append("-I%s" % dir) - return pp_opts - - -def gen_lib_options (compiler, library_dirs, runtime_library_dirs, libraries): - """Generate linker options for searching library directories and - linking with specific libraries. 'libraries' and 'library_dirs' are, - respectively, lists of library names (not filenames!) and search - directories. Returns a list of command-line options suitable for use - with some compiler (depending on the two format strings passed in). - """ - lib_opts = [] - - for dir in library_dirs: - lib_opts.append(compiler.library_dir_option(dir)) - - for dir in runtime_library_dirs: - opt = compiler.runtime_library_dir_option(dir) - if isinstance(opt, list): - lib_opts = lib_opts + opt - else: - lib_opts.append(opt) - - # XXX it's important that we *not* remove redundant library mentions! - # sometimes you really do have to say "-lfoo -lbar -lfoo" in order to - # resolve all symbols. I just hope we never have to say "-lfoo obj.o - # -lbar" to get things to work -- that's certainly a possibility, but a - # pretty nasty way to arrange your C code. - - for lib in libraries: - (lib_dir, lib_name) = os.path.split(lib) - if lib_dir: - lib_file = compiler.find_library_file([lib_dir], lib_name) - if lib_file: - lib_opts.append(lib_file) - else: - compiler.warn("no library file corresponding to " - "'%s' found (skipping)" % lib) - else: - lib_opts.append(compiler.library_option (lib)) - return lib_opts diff --git a/Lib/distutils/cmd.py b/Lib/distutils/cmd.py deleted file mode 100644 index 939f7959457..00000000000 --- a/Lib/distutils/cmd.py +++ /dev/null @@ -1,434 +0,0 @@ -"""distutils.cmd - -Provides the Command class, the base class for the command classes -in the distutils.command package. -""" - -import sys, os, re -from distutils.errors import DistutilsOptionError -from distutils import util, dir_util, file_util, archive_util, dep_util -from distutils import log - -class Command: - """Abstract base class for defining command classes, the "worker bees" - of the Distutils. A useful analogy for command classes is to think of - them as subroutines with local variables called "options". The options - are "declared" in 'initialize_options()' and "defined" (given their - final values, aka "finalized") in 'finalize_options()', both of which - must be defined by every command class. The distinction between the - two is necessary because option values might come from the outside - world (command line, config file, ...), and any options dependent on - other options must be computed *after* these outside influences have - been processed -- hence 'finalize_options()'. The "body" of the - subroutine, where it does all its work based on the values of its - options, is the 'run()' method, which must also be implemented by every - command class. - """ - - # 'sub_commands' formalizes the notion of a "family" of commands, - # eg. "install" as the parent with sub-commands "install_lib", - # "install_headers", etc. The parent of a family of commands - # defines 'sub_commands' as a class attribute; it's a list of - # (command_name : string, predicate : unbound_method | string | None) - # tuples, where 'predicate' is a method of the parent command that - # determines whether the corresponding command is applicable in the - # current situation. (Eg. we "install_headers" is only applicable if - # we have any C header files to install.) If 'predicate' is None, - # that command is always applicable. - # - # 'sub_commands' is usually defined at the *end* of a class, because - # predicates can be unbound methods, so they must already have been - # defined. The canonical example is the "install" command. - sub_commands = [] - - - # -- Creation/initialization methods ------------------------------- - - def __init__(self, dist): - """Create and initialize a new Command object. Most importantly, - invokes the 'initialize_options()' method, which is the real - initializer and depends on the actual command being - instantiated. - """ - # late import because of mutual dependence between these classes - from distutils.dist import Distribution - - if not isinstance(dist, Distribution): - raise TypeError("dist must be a Distribution instance") - if self.__class__ is Command: - raise RuntimeError("Command is an abstract class") - - self.distribution = dist - self.initialize_options() - - # Per-command versions of the global flags, so that the user can - # customize Distutils' behaviour command-by-command and let some - # commands fall back on the Distribution's behaviour. None means - # "not defined, check self.distribution's copy", while 0 or 1 mean - # false and true (duh). Note that this means figuring out the real - # value of each flag is a touch complicated -- hence "self._dry_run" - # will be handled by __getattr__, below. - # XXX This needs to be fixed. - self._dry_run = None - - # verbose is largely ignored, but needs to be set for - # backwards compatibility (I think)? - self.verbose = dist.verbose - - # Some commands define a 'self.force' option to ignore file - # timestamps, but methods defined *here* assume that - # 'self.force' exists for all commands. So define it here - # just to be safe. - self.force = None - - # The 'help' flag is just used for command-line parsing, so - # none of that complicated bureaucracy is needed. - self.help = 0 - - # 'finalized' records whether or not 'finalize_options()' has been - # called. 'finalize_options()' itself should not pay attention to - # this flag: it is the business of 'ensure_finalized()', which - # always calls 'finalize_options()', to respect/update it. - self.finalized = 0 - - # XXX A more explicit way to customize dry_run would be better. - def __getattr__(self, attr): - if attr == 'dry_run': - myval = getattr(self, "_" + attr) - if myval is None: - return getattr(self.distribution, attr) - else: - return myval - else: - raise AttributeError(attr) - - def ensure_finalized(self): - if not self.finalized: - self.finalize_options() - self.finalized = 1 - - # Subclasses must define: - # initialize_options() - # provide default values for all options; may be customized by - # setup script, by options from config file(s), or by command-line - # options - # finalize_options() - # decide on the final values for all options; this is called - # after all possible intervention from the outside world - # (command-line, option file, etc.) has been processed - # run() - # run the command: do whatever it is we're here to do, - # controlled by the command's various option values - - def initialize_options(self): - """Set default values for all the options that this command - supports. Note that these defaults may be overridden by other - commands, by the setup script, by config files, or by the - command-line. Thus, this is not the place to code dependencies - between options; generally, 'initialize_options()' implementations - are just a bunch of "self.foo = None" assignments. - - This method must be implemented by all command classes. - """ - raise RuntimeError("abstract method -- subclass %s must override" - % self.__class__) - - def finalize_options(self): - """Set final values for all the options that this command supports. - This is always called as late as possible, ie. after any option - assignments from the command-line or from other commands have been - done. Thus, this is the place to code option dependencies: if - 'foo' depends on 'bar', then it is safe to set 'foo' from 'bar' as - long as 'foo' still has the same value it was assigned in - 'initialize_options()'. - - This method must be implemented by all command classes. - """ - raise RuntimeError("abstract method -- subclass %s must override" - % self.__class__) - - - def dump_options(self, header=None, indent=""): - from distutils.fancy_getopt import longopt_xlate - if header is None: - header = "command options for '%s':" % self.get_command_name() - self.announce(indent + header, level=log.INFO) - indent = indent + " " - for (option, _, _) in self.user_options: - option = option.translate(longopt_xlate) - if option[-1] == "=": - option = option[:-1] - value = getattr(self, option) - self.announce(indent + "%s = %s" % (option, value), - level=log.INFO) - - def run(self): - """A command's raison d'etre: carry out the action it exists to - perform, controlled by the options initialized in - 'initialize_options()', customized by other commands, the setup - script, the command-line, and config files, and finalized in - 'finalize_options()'. All terminal output and filesystem - interaction should be done by 'run()'. - - This method must be implemented by all command classes. - """ - raise RuntimeError("abstract method -- subclass %s must override" - % self.__class__) - - def announce(self, msg, level=1): - """If the current verbosity level is of greater than or equal to - 'level' print 'msg' to stdout. - """ - log.log(level, msg) - - def debug_print(self, msg): - """Print 'msg' to stdout if the global DEBUG (taken from the - DISTUTILS_DEBUG environment variable) flag is true. - """ - from distutils.debug import DEBUG - if DEBUG: - print(msg) - sys.stdout.flush() - - - # -- Option validation methods ------------------------------------- - # (these are very handy in writing the 'finalize_options()' method) - # - # NB. the general philosophy here is to ensure that a particular option - # value meets certain type and value constraints. If not, we try to - # force it into conformance (eg. if we expect a list but have a string, - # split the string on comma and/or whitespace). If we can't force the - # option into conformance, raise DistutilsOptionError. Thus, command - # classes need do nothing more than (eg.) - # self.ensure_string_list('foo') - # and they can be guaranteed that thereafter, self.foo will be - # a list of strings. - - def _ensure_stringlike(self, option, what, default=None): - val = getattr(self, option) - if val is None: - setattr(self, option, default) - return default - elif not isinstance(val, str): - raise DistutilsOptionError("'%s' must be a %s (got `%s`)" - % (option, what, val)) - return val - - def ensure_string(self, option, default=None): - """Ensure that 'option' is a string; if not defined, set it to - 'default'. - """ - self._ensure_stringlike(option, "string", default) - - def ensure_string_list(self, option): - r"""Ensure that 'option' is a list of strings. If 'option' is - currently a string, we split it either on /,\s*/ or /\s+/, so - "foo bar baz", "foo,bar,baz", and "foo, bar baz" all become - ["foo", "bar", "baz"]. - """ - val = getattr(self, option) - if val is None: - return - elif isinstance(val, str): - setattr(self, option, re.split(r',\s*|\s+', val)) - else: - if isinstance(val, list): - ok = all(isinstance(v, str) for v in val) - else: - ok = False - if not ok: - raise DistutilsOptionError( - "'%s' must be a list of strings (got %r)" - % (option, val)) - - def _ensure_tested_string(self, option, tester, what, error_fmt, - default=None): - val = self._ensure_stringlike(option, what, default) - if val is not None and not tester(val): - raise DistutilsOptionError(("error in '%s' option: " + error_fmt) - % (option, val)) - - def ensure_filename(self, option): - """Ensure that 'option' is the name of an existing file.""" - self._ensure_tested_string(option, os.path.isfile, - "filename", - "'%s' does not exist or is not a file") - - def ensure_dirname(self, option): - self._ensure_tested_string(option, os.path.isdir, - "directory name", - "'%s' does not exist or is not a directory") - - - # -- Convenience methods for commands ------------------------------ - - def get_command_name(self): - if hasattr(self, 'command_name'): - return self.command_name - else: - return self.__class__.__name__ - - def set_undefined_options(self, src_cmd, *option_pairs): - """Set the values of any "undefined" options from corresponding - option values in some other command object. "Undefined" here means - "is None", which is the convention used to indicate that an option - has not been changed between 'initialize_options()' and - 'finalize_options()'. Usually called from 'finalize_options()' for - options that depend on some other command rather than another - option of the same command. 'src_cmd' is the other command from - which option values will be taken (a command object will be created - for it if necessary); the remaining arguments are - '(src_option,dst_option)' tuples which mean "take the value of - 'src_option' in the 'src_cmd' command object, and copy it to - 'dst_option' in the current command object". - """ - # Option_pairs: list of (src_option, dst_option) tuples - src_cmd_obj = self.distribution.get_command_obj(src_cmd) - src_cmd_obj.ensure_finalized() - for (src_option, dst_option) in option_pairs: - if getattr(self, dst_option) is None: - setattr(self, dst_option, getattr(src_cmd_obj, src_option)) - - def get_finalized_command(self, command, create=1): - """Wrapper around Distribution's 'get_command_obj()' method: find - (create if necessary and 'create' is true) the command object for - 'command', call its 'ensure_finalized()' method, and return the - finalized command object. - """ - cmd_obj = self.distribution.get_command_obj(command, create) - cmd_obj.ensure_finalized() - return cmd_obj - - # XXX rename to 'get_reinitialized_command()'? (should do the - # same in dist.py, if so) - def reinitialize_command(self, command, reinit_subcommands=0): - return self.distribution.reinitialize_command(command, - reinit_subcommands) - - def run_command(self, command): - """Run some other command: uses the 'run_command()' method of - Distribution, which creates and finalizes the command object if - necessary and then invokes its 'run()' method. - """ - self.distribution.run_command(command) - - def get_sub_commands(self): - """Determine the sub-commands that are relevant in the current - distribution (ie., that need to be run). This is based on the - 'sub_commands' class attribute: each tuple in that list may include - a method that we call to determine if the subcommand needs to be - run for the current distribution. Return a list of command names. - """ - commands = [] - for (cmd_name, method) in self.sub_commands: - if method is None or method(self): - commands.append(cmd_name) - return commands - - - # -- External world manipulation ----------------------------------- - - def warn(self, msg): - log.warn("warning: %s: %s\n", self.get_command_name(), msg) - - def execute(self, func, args, msg=None, level=1): - util.execute(func, args, msg, dry_run=self.dry_run) - - def mkpath(self, name, mode=0o777): - dir_util.mkpath(name, mode, dry_run=self.dry_run) - - def copy_file(self, infile, outfile, preserve_mode=1, preserve_times=1, - link=None, level=1): - """Copy a file respecting verbose, dry-run and force flags. (The - former two default to whatever is in the Distribution object, and - the latter defaults to false for commands that don't define it.)""" - return file_util.copy_file(infile, outfile, preserve_mode, - preserve_times, not self.force, link, - dry_run=self.dry_run) - - def copy_tree(self, infile, outfile, preserve_mode=1, preserve_times=1, - preserve_symlinks=0, level=1): - """Copy an entire directory tree respecting verbose, dry-run, - and force flags. - """ - return dir_util.copy_tree(infile, outfile, preserve_mode, - preserve_times, preserve_symlinks, - not self.force, dry_run=self.dry_run) - - def move_file (self, src, dst, level=1): - """Move a file respecting dry-run flag.""" - return file_util.move_file(src, dst, dry_run=self.dry_run) - - def spawn(self, cmd, search_path=1, level=1): - """Spawn an external command respecting dry-run flag.""" - from distutils.spawn import spawn - spawn(cmd, search_path, dry_run=self.dry_run) - - def make_archive(self, base_name, format, root_dir=None, base_dir=None, - owner=None, group=None): - return archive_util.make_archive(base_name, format, root_dir, base_dir, - dry_run=self.dry_run, - owner=owner, group=group) - - def make_file(self, infiles, outfile, func, args, - exec_msg=None, skip_msg=None, level=1): - """Special case of 'execute()' for operations that process one or - more input files and generate one output file. Works just like - 'execute()', except the operation is skipped and a different - message printed if 'outfile' already exists and is newer than all - files listed in 'infiles'. If the command defined 'self.force', - and it is true, then the command is unconditionally run -- does no - timestamp checks. - """ - if skip_msg is None: - skip_msg = "skipping %s (inputs unchanged)" % outfile - - # Allow 'infiles' to be a single string - if isinstance(infiles, str): - infiles = (infiles,) - elif not isinstance(infiles, (list, tuple)): - raise TypeError( - "'infiles' must be a string, or a list or tuple of strings") - - if exec_msg is None: - exec_msg = "generating %s from %s" % (outfile, ', '.join(infiles)) - - # If 'outfile' must be regenerated (either because it doesn't - # exist, is out-of-date, or the 'force' flag is true) then - # perform the action that presumably regenerates it - if self.force or dep_util.newer_group(infiles, outfile): - self.execute(func, args, exec_msg, level) - # Otherwise, print the "skip" message - else: - log.debug(skip_msg) - -# XXX 'install_misc' class not currently used -- it was the base class for -# both 'install_scripts' and 'install_data', but they outgrew it. It might -# still be useful for 'install_headers', though, so I'm keeping it around -# for the time being. - -class install_misc(Command): - """Common base class for installing some files in a subdirectory. - Currently used by install_data and install_scripts. - """ - - user_options = [('install-dir=', 'd', "directory to install the files to")] - - def initialize_options (self): - self.install_dir = None - self.outfiles = [] - - def _install_dir_from(self, dirname): - self.set_undefined_options('install', (dirname, 'install_dir')) - - def _copy_files(self, filelist): - self.outfiles = [] - if not filelist: - return - self.mkpath(self.install_dir) - for f in filelist: - self.copy_file(f, self.install_dir) - self.outfiles.append(os.path.join(self.install_dir, f)) - - def get_outputs(self): - return self.outfiles diff --git a/Lib/distutils/command/__init__.py b/Lib/distutils/command/__init__.py deleted file mode 100644 index 481eea9fd4b..00000000000 --- a/Lib/distutils/command/__init__.py +++ /dev/null @@ -1,31 +0,0 @@ -"""distutils.command - -Package containing implementation of all the standard Distutils -commands.""" - -__all__ = ['build', - 'build_py', - 'build_ext', - 'build_clib', - 'build_scripts', - 'clean', - 'install', - 'install_lib', - 'install_headers', - 'install_scripts', - 'install_data', - 'sdist', - 'register', - 'bdist', - 'bdist_dumb', - 'bdist_rpm', - 'bdist_wininst', - 'check', - 'upload', - # These two are reserved for future use: - #'bdist_sdux', - #'bdist_pkgtool', - # Note: - # bdist_packager is not included because it only provides - # an abstract base class - ] diff --git a/Lib/distutils/command/bdist.py b/Lib/distutils/command/bdist.py deleted file mode 100644 index 014871d280e..00000000000 --- a/Lib/distutils/command/bdist.py +++ /dev/null @@ -1,143 +0,0 @@ -"""distutils.command.bdist - -Implements the Distutils 'bdist' command (create a built [binary] -distribution).""" - -import os -from distutils.core import Command -from distutils.errors import * -from distutils.util import get_platform - - -def show_formats(): - """Print list of available formats (arguments to "--format" option). - """ - from distutils.fancy_getopt import FancyGetopt - formats = [] - for format in bdist.format_commands: - formats.append(("formats=" + format, None, - bdist.format_command[format][1])) - pretty_printer = FancyGetopt(formats) - pretty_printer.print_help("List of available distribution formats:") - - -class bdist(Command): - - description = "create a built (binary) distribution" - - user_options = [('bdist-base=', 'b', - "temporary directory for creating built distributions"), - ('plat-name=', 'p', - "platform name to embed in generated filenames " - "(default: %s)" % get_platform()), - ('formats=', None, - "formats for distribution (comma-separated list)"), - ('dist-dir=', 'd', - "directory to put final built distributions in " - "[default: dist]"), - ('skip-build', None, - "skip rebuilding everything (for testing/debugging)"), - ('owner=', 'u', - "Owner name used when creating a tar file" - " [default: current user]"), - ('group=', 'g', - "Group name used when creating a tar file" - " [default: current group]"), - ] - - boolean_options = ['skip-build'] - - help_options = [ - ('help-formats', None, - "lists available distribution formats", show_formats), - ] - - # The following commands do not take a format option from bdist - no_format_option = ('bdist_rpm',) - - # This won't do in reality: will need to distinguish RPM-ish Linux, - # Debian-ish Linux, Solaris, FreeBSD, ..., Windows, Mac OS. - default_format = {'posix': 'gztar', - 'nt': 'zip'} - - # Establish the preferred order (for the --help-formats option). - format_commands = ['rpm', 'gztar', 'bztar', 'xztar', 'ztar', 'tar', - 'wininst', 'zip', 'msi'] - - # And the real information. - format_command = {'rpm': ('bdist_rpm', "RPM distribution"), - 'gztar': ('bdist_dumb', "gzip'ed tar file"), - 'bztar': ('bdist_dumb', "bzip2'ed tar file"), - 'xztar': ('bdist_dumb', "xz'ed tar file"), - 'ztar': ('bdist_dumb', "compressed tar file"), - 'tar': ('bdist_dumb', "tar file"), - 'wininst': ('bdist_wininst', - "Windows executable installer"), - 'zip': ('bdist_dumb', "ZIP file"), - 'msi': ('bdist_msi', "Microsoft Installer") - } - - - def initialize_options(self): - self.bdist_base = None - self.plat_name = None - self.formats = None - self.dist_dir = None - self.skip_build = 0 - self.group = None - self.owner = None - - def finalize_options(self): - # have to finalize 'plat_name' before 'bdist_base' - if self.plat_name is None: - if self.skip_build: - self.plat_name = get_platform() - else: - self.plat_name = self.get_finalized_command('build').plat_name - - # 'bdist_base' -- parent of per-built-distribution-format - # temporary directories (eg. we'll probably have - # "build/bdist./dumb", "build/bdist./rpm", etc.) - if self.bdist_base is None: - build_base = self.get_finalized_command('build').build_base - self.bdist_base = os.path.join(build_base, - 'bdist.' + self.plat_name) - - self.ensure_string_list('formats') - if self.formats is None: - try: - self.formats = [self.default_format[os.name]] - except KeyError: - raise DistutilsPlatformError( - "don't know how to create built distributions " - "on platform %s" % os.name) - - if self.dist_dir is None: - self.dist_dir = "dist" - - def run(self): - # Figure out which sub-commands we need to run. - commands = [] - for format in self.formats: - try: - commands.append(self.format_command[format][0]) - except KeyError: - raise DistutilsOptionError("invalid format '%s'" % format) - - # Reinitialize and run each command. - for i in range(len(self.formats)): - cmd_name = commands[i] - sub_cmd = self.reinitialize_command(cmd_name) - if cmd_name not in self.no_format_option: - sub_cmd.format = self.formats[i] - - # passing the owner and group names for tar archiving - if cmd_name == 'bdist_dumb': - sub_cmd.owner = self.owner - sub_cmd.group = self.group - - # If we're going to need to run this command again, tell it to - # keep its temporary files around so subsequent runs go faster. - if cmd_name in commands[i+1:]: - sub_cmd.keep_temp = 1 - self.run_command(cmd_name) diff --git a/Lib/distutils/command/bdist_dumb.py b/Lib/distutils/command/bdist_dumb.py deleted file mode 100644 index f0d6b5b8cd8..00000000000 --- a/Lib/distutils/command/bdist_dumb.py +++ /dev/null @@ -1,123 +0,0 @@ -"""distutils.command.bdist_dumb - -Implements the Distutils 'bdist_dumb' command (create a "dumb" built -distribution -- i.e., just an archive to be unpacked under $prefix or -$exec_prefix).""" - -import os -from distutils.core import Command -from distutils.util import get_platform -from distutils.dir_util import remove_tree, ensure_relative -from distutils.errors import * -from distutils.sysconfig import get_python_version -from distutils import log - -class bdist_dumb(Command): - - description = "create a \"dumb\" built distribution" - - user_options = [('bdist-dir=', 'd', - "temporary directory for creating the distribution"), - ('plat-name=', 'p', - "platform name to embed in generated filenames " - "(default: %s)" % get_platform()), - ('format=', 'f', - "archive format to create (tar, gztar, bztar, xztar, " - "ztar, zip)"), - ('keep-temp', 'k', - "keep the pseudo-installation tree around after " + - "creating the distribution archive"), - ('dist-dir=', 'd', - "directory to put final built distributions in"), - ('skip-build', None, - "skip rebuilding everything (for testing/debugging)"), - ('relative', None, - "build the archive using relative paths " - "(default: false)"), - ('owner=', 'u', - "Owner name used when creating a tar file" - " [default: current user]"), - ('group=', 'g', - "Group name used when creating a tar file" - " [default: current group]"), - ] - - boolean_options = ['keep-temp', 'skip-build', 'relative'] - - default_format = { 'posix': 'gztar', - 'nt': 'zip' } - - def initialize_options(self): - self.bdist_dir = None - self.plat_name = None - self.format = None - self.keep_temp = 0 - self.dist_dir = None - self.skip_build = None - self.relative = 0 - self.owner = None - self.group = None - - def finalize_options(self): - if self.bdist_dir is None: - bdist_base = self.get_finalized_command('bdist').bdist_base - self.bdist_dir = os.path.join(bdist_base, 'dumb') - - if self.format is None: - try: - self.format = self.default_format[os.name] - except KeyError: - raise DistutilsPlatformError( - "don't know how to create dumb built distributions " - "on platform %s" % os.name) - - self.set_undefined_options('bdist', - ('dist_dir', 'dist_dir'), - ('plat_name', 'plat_name'), - ('skip_build', 'skip_build')) - - def run(self): - if not self.skip_build: - self.run_command('build') - - install = self.reinitialize_command('install', reinit_subcommands=1) - install.root = self.bdist_dir - install.skip_build = self.skip_build - install.warn_dir = 0 - - log.info("installing to %s", self.bdist_dir) - self.run_command('install') - - # And make an archive relative to the root of the - # pseudo-installation tree. - archive_basename = "%s.%s" % (self.distribution.get_fullname(), - self.plat_name) - - pseudoinstall_root = os.path.join(self.dist_dir, archive_basename) - if not self.relative: - archive_root = self.bdist_dir - else: - if (self.distribution.has_ext_modules() and - (install.install_base != install.install_platbase)): - raise DistutilsPlatformError( - "can't make a dumb built distribution where " - "base and platbase are different (%s, %s)" - % (repr(install.install_base), - repr(install.install_platbase))) - else: - archive_root = os.path.join(self.bdist_dir, - ensure_relative(install.install_base)) - - # Make the archive - filename = self.make_archive(pseudoinstall_root, - self.format, root_dir=archive_root, - owner=self.owner, group=self.group) - if self.distribution.has_ext_modules(): - pyversion = get_python_version() - else: - pyversion = 'any' - self.distribution.dist_files.append(('bdist_dumb', pyversion, - filename)) - - if not self.keep_temp: - remove_tree(self.bdist_dir, dry_run=self.dry_run) diff --git a/Lib/distutils/command/bdist_msi.py b/Lib/distutils/command/bdist_msi.py deleted file mode 100644 index 80104c372d9..00000000000 --- a/Lib/distutils/command/bdist_msi.py +++ /dev/null @@ -1,741 +0,0 @@ -# Copyright (C) 2005, 2006 Martin von Löwis -# Licensed to PSF under a Contributor Agreement. -# The bdist_wininst command proper -# based on bdist_wininst -""" -Implements the bdist_msi command. -""" - -import sys, os -from distutils.core import Command -from distutils.dir_util import remove_tree -from distutils.sysconfig import get_python_version -from distutils.version import StrictVersion -from distutils.errors import DistutilsOptionError -from distutils.util import get_platform -from distutils import log -import msilib -from msilib import schema, sequence, text -from msilib import Directory, Feature, Dialog, add_data - -class PyDialog(Dialog): - """Dialog class with a fixed layout: controls at the top, then a ruler, - then a list of buttons: back, next, cancel. Optionally a bitmap at the - left.""" - def __init__(self, *args, **kw): - """Dialog(database, name, x, y, w, h, attributes, title, first, - default, cancel, bitmap=true)""" - Dialog.__init__(self, *args) - ruler = self.h - 36 - bmwidth = 152*ruler/328 - #if kw.get("bitmap", True): - # self.bitmap("Bitmap", 0, 0, bmwidth, ruler, "PythonWin") - self.line("BottomLine", 0, ruler, self.w, 0) - - def title(self, title): - "Set the title text of the dialog at the top." - # name, x, y, w, h, flags=Visible|Enabled|Transparent|NoPrefix, - # text, in VerdanaBold10 - self.text("Title", 15, 10, 320, 60, 0x30003, - r"{\VerdanaBold10}%s" % title) - - def back(self, title, next, name = "Back", active = 1): - """Add a back button with a given title, the tab-next button, - its name in the Control table, possibly initially disabled. - - Return the button, so that events can be associated""" - if active: - flags = 3 # Visible|Enabled - else: - flags = 1 # Visible - return self.pushbutton(name, 180, self.h-27 , 56, 17, flags, title, next) - - def cancel(self, title, next, name = "Cancel", active = 1): - """Add a cancel button with a given title, the tab-next button, - its name in the Control table, possibly initially disabled. - - Return the button, so that events can be associated""" - if active: - flags = 3 # Visible|Enabled - else: - flags = 1 # Visible - return self.pushbutton(name, 304, self.h-27, 56, 17, flags, title, next) - - def next(self, title, next, name = "Next", active = 1): - """Add a Next button with a given title, the tab-next button, - its name in the Control table, possibly initially disabled. - - Return the button, so that events can be associated""" - if active: - flags = 3 # Visible|Enabled - else: - flags = 1 # Visible - return self.pushbutton(name, 236, self.h-27, 56, 17, flags, title, next) - - def xbutton(self, name, title, next, xpos): - """Add a button with a given title, the tab-next button, - its name in the Control table, giving its x position; the - y-position is aligned with the other buttons. - - Return the button, so that events can be associated""" - return self.pushbutton(name, int(self.w*xpos - 28), self.h-27, 56, 17, 3, title, next) - -class bdist_msi(Command): - - description = "create a Microsoft Installer (.msi) binary distribution" - - user_options = [('bdist-dir=', None, - "temporary directory for creating the distribution"), - ('plat-name=', 'p', - "platform name to embed in generated filenames " - "(default: %s)" % get_platform()), - ('keep-temp', 'k', - "keep the pseudo-installation tree around after " + - "creating the distribution archive"), - ('target-version=', None, - "require a specific python version" + - " on the target system"), - ('no-target-compile', 'c', - "do not compile .py to .pyc on the target system"), - ('no-target-optimize', 'o', - "do not compile .py to .pyo (optimized) " - "on the target system"), - ('dist-dir=', 'd', - "directory to put final built distributions in"), - ('skip-build', None, - "skip rebuilding everything (for testing/debugging)"), - ('install-script=', None, - "basename of installation script to be run after " - "installation or before deinstallation"), - ('pre-install-script=', None, - "Fully qualified filename of a script to be run before " - "any files are installed. This script need not be in the " - "distribution"), - ] - - boolean_options = ['keep-temp', 'no-target-compile', 'no-target-optimize', - 'skip-build'] - - all_versions = ['2.0', '2.1', '2.2', '2.3', '2.4', - '2.5', '2.6', '2.7', '2.8', '2.9', - '3.0', '3.1', '3.2', '3.3', '3.4', - '3.5', '3.6', '3.7', '3.8', '3.9'] - other_version = 'X' - - def initialize_options(self): - self.bdist_dir = None - self.plat_name = None - self.keep_temp = 0 - self.no_target_compile = 0 - self.no_target_optimize = 0 - self.target_version = None - self.dist_dir = None - self.skip_build = None - self.install_script = None - self.pre_install_script = None - self.versions = None - - def finalize_options(self): - self.set_undefined_options('bdist', ('skip_build', 'skip_build')) - - if self.bdist_dir is None: - bdist_base = self.get_finalized_command('bdist').bdist_base - self.bdist_dir = os.path.join(bdist_base, 'msi') - - short_version = get_python_version() - if (not self.target_version) and self.distribution.has_ext_modules(): - self.target_version = short_version - - if self.target_version: - self.versions = [self.target_version] - if not self.skip_build and self.distribution.has_ext_modules()\ - and self.target_version != short_version: - raise DistutilsOptionError( - "target version can only be %s, or the '--skip-build'" - " option must be specified" % (short_version,)) - else: - self.versions = list(self.all_versions) - - self.set_undefined_options('bdist', - ('dist_dir', 'dist_dir'), - ('plat_name', 'plat_name'), - ) - - if self.pre_install_script: - raise DistutilsOptionError( - "the pre-install-script feature is not yet implemented") - - if self.install_script: - for script in self.distribution.scripts: - if self.install_script == os.path.basename(script): - break - else: - raise DistutilsOptionError( - "install_script '%s' not found in scripts" - % self.install_script) - self.install_script_key = None - - def run(self): - if not self.skip_build: - self.run_command('build') - - install = self.reinitialize_command('install', reinit_subcommands=1) - install.prefix = self.bdist_dir - install.skip_build = self.skip_build - install.warn_dir = 0 - - install_lib = self.reinitialize_command('install_lib') - # we do not want to include pyc or pyo files - install_lib.compile = 0 - install_lib.optimize = 0 - - if self.distribution.has_ext_modules(): - # If we are building an installer for a Python version other - # than the one we are currently running, then we need to ensure - # our build_lib reflects the other Python version rather than ours. - # Note that for target_version!=sys.version, we must have skipped the - # build step, so there is no issue with enforcing the build of this - # version. - target_version = self.target_version - if not target_version: - assert self.skip_build, "Should have already checked this" - target_version = '%d.%d' % sys.version_info[:2] - plat_specifier = ".%s-%s" % (self.plat_name, target_version) - build = self.get_finalized_command('build') - build.build_lib = os.path.join(build.build_base, - 'lib' + plat_specifier) - - log.info("installing to %s", self.bdist_dir) - install.ensure_finalized() - - # avoid warning of 'install_lib' about installing - # into a directory not in sys.path - sys.path.insert(0, os.path.join(self.bdist_dir, 'PURELIB')) - - install.run() - - del sys.path[0] - - self.mkpath(self.dist_dir) - fullname = self.distribution.get_fullname() - installer_name = self.get_installer_filename(fullname) - installer_name = os.path.abspath(installer_name) - if os.path.exists(installer_name): os.unlink(installer_name) - - metadata = self.distribution.metadata - author = metadata.author - if not author: - author = metadata.maintainer - if not author: - author = "UNKNOWN" - version = metadata.get_version() - # ProductVersion must be strictly numeric - # XXX need to deal with prerelease versions - sversion = "%d.%d.%d" % StrictVersion(version).version - # Prefix ProductName with Python x.y, so that - # it sorts together with the other Python packages - # in Add-Remove-Programs (APR) - fullname = self.distribution.get_fullname() - if self.target_version: - product_name = "Python %s %s" % (self.target_version, fullname) - else: - product_name = "Python %s" % (fullname) - self.db = msilib.init_database(installer_name, schema, - product_name, msilib.gen_uuid(), - sversion, author) - msilib.add_tables(self.db, sequence) - props = [('DistVersion', version)] - email = metadata.author_email or metadata.maintainer_email - if email: - props.append(("ARPCONTACT", email)) - if metadata.url: - props.append(("ARPURLINFOABOUT", metadata.url)) - if props: - add_data(self.db, 'Property', props) - - self.add_find_python() - self.add_files() - self.add_scripts() - self.add_ui() - self.db.Commit() - - if hasattr(self.distribution, 'dist_files'): - tup = 'bdist_msi', self.target_version or 'any', fullname - self.distribution.dist_files.append(tup) - - if not self.keep_temp: - remove_tree(self.bdist_dir, dry_run=self.dry_run) - - def add_files(self): - db = self.db - cab = msilib.CAB("distfiles") - rootdir = os.path.abspath(self.bdist_dir) - - root = Directory(db, cab, None, rootdir, "TARGETDIR", "SourceDir") - f = Feature(db, "Python", "Python", "Everything", - 0, 1, directory="TARGETDIR") - - items = [(f, root, '')] - for version in self.versions + [self.other_version]: - target = "TARGETDIR" + version - name = default = "Python" + version - desc = "Everything" - if version is self.other_version: - title = "Python from another location" - level = 2 - else: - title = "Python %s from registry" % version - level = 1 - f = Feature(db, name, title, desc, 1, level, directory=target) - dir = Directory(db, cab, root, rootdir, target, default) - items.append((f, dir, version)) - db.Commit() - - seen = {} - for feature, dir, version in items: - todo = [dir] - while todo: - dir = todo.pop() - for file in os.listdir(dir.absolute): - afile = os.path.join(dir.absolute, file) - if os.path.isdir(afile): - short = "%s|%s" % (dir.make_short(file), file) - default = file + version - newdir = Directory(db, cab, dir, file, default, short) - todo.append(newdir) - else: - if not dir.component: - dir.start_component(dir.logical, feature, 0) - if afile not in seen: - key = seen[afile] = dir.add_file(file) - if file==self.install_script: - if self.install_script_key: - raise DistutilsOptionError( - "Multiple files with name %s" % file) - self.install_script_key = '[#%s]' % key - else: - key = seen[afile] - add_data(self.db, "DuplicateFile", - [(key + version, dir.component, key, None, dir.logical)]) - db.Commit() - cab.commit(db) - - def add_find_python(self): - """Adds code to the installer to compute the location of Python. - - Properties PYTHON.MACHINE.X.Y and PYTHON.USER.X.Y will be set from the - registry for each version of Python. - - Properties TARGETDIRX.Y will be set from PYTHON.USER.X.Y if defined, - else from PYTHON.MACHINE.X.Y. - - Properties PYTHONX.Y will be set to TARGETDIRX.Y\\python.exe""" - - start = 402 - for ver in self.versions: - install_path = r"SOFTWARE\Python\PythonCore\%s\InstallPath" % ver - machine_reg = "python.machine." + ver - user_reg = "python.user." + ver - machine_prop = "PYTHON.MACHINE." + ver - user_prop = "PYTHON.USER." + ver - machine_action = "PythonFromMachine" + ver - user_action = "PythonFromUser" + ver - exe_action = "PythonExe" + ver - target_dir_prop = "TARGETDIR" + ver - exe_prop = "PYTHON" + ver - if msilib.Win64: - # type: msidbLocatorTypeRawValue + msidbLocatorType64bit - Type = 2+16 - else: - Type = 2 - add_data(self.db, "RegLocator", - [(machine_reg, 2, install_path, None, Type), - (user_reg, 1, install_path, None, Type)]) - add_data(self.db, "AppSearch", - [(machine_prop, machine_reg), - (user_prop, user_reg)]) - add_data(self.db, "CustomAction", - [(machine_action, 51+256, target_dir_prop, "[" + machine_prop + "]"), - (user_action, 51+256, target_dir_prop, "[" + user_prop + "]"), - (exe_action, 51+256, exe_prop, "[" + target_dir_prop + "]\\python.exe"), - ]) - add_data(self.db, "InstallExecuteSequence", - [(machine_action, machine_prop, start), - (user_action, user_prop, start + 1), - (exe_action, None, start + 2), - ]) - add_data(self.db, "InstallUISequence", - [(machine_action, machine_prop, start), - (user_action, user_prop, start + 1), - (exe_action, None, start + 2), - ]) - add_data(self.db, "Condition", - [("Python" + ver, 0, "NOT TARGETDIR" + ver)]) - start += 4 - assert start < 500 - - def add_scripts(self): - if self.install_script: - start = 6800 - for ver in self.versions + [self.other_version]: - install_action = "install_script." + ver - exe_prop = "PYTHON" + ver - add_data(self.db, "CustomAction", - [(install_action, 50, exe_prop, self.install_script_key)]) - add_data(self.db, "InstallExecuteSequence", - [(install_action, "&Python%s=3" % ver, start)]) - start += 1 - # XXX pre-install scripts are currently refused in finalize_options() - # but if this feature is completed, it will also need to add - # entries for each version as the above code does - if self.pre_install_script: - scriptfn = os.path.join(self.bdist_dir, "preinstall.bat") - f = open(scriptfn, "w") - # The batch file will be executed with [PYTHON], so that %1 - # is the path to the Python interpreter; %0 will be the path - # of the batch file. - # rem =""" - # %1 %0 - # exit - # """ - # - f.write('rem ="""\n%1 %0\nexit\n"""\n') - f.write(open(self.pre_install_script).read()) - f.close() - add_data(self.db, "Binary", - [("PreInstall", msilib.Binary(scriptfn)) - ]) - add_data(self.db, "CustomAction", - [("PreInstall", 2, "PreInstall", None) - ]) - add_data(self.db, "InstallExecuteSequence", - [("PreInstall", "NOT Installed", 450)]) - - - def add_ui(self): - db = self.db - x = y = 50 - w = 370 - h = 300 - title = "[ProductName] Setup" - - # see "Dialog Style Bits" - modal = 3 # visible | modal - modeless = 1 # visible - track_disk_space = 32 - - # UI customization properties - add_data(db, "Property", - # See "DefaultUIFont Property" - [("DefaultUIFont", "DlgFont8"), - # See "ErrorDialog Style Bit" - ("ErrorDialog", "ErrorDlg"), - ("Progress1", "Install"), # modified in maintenance type dlg - ("Progress2", "installs"), - ("MaintenanceForm_Action", "Repair"), - # possible values: ALL, JUSTME - ("WhichUsers", "ALL") - ]) - - # Fonts, see "TextStyle Table" - add_data(db, "TextStyle", - [("DlgFont8", "Tahoma", 9, None, 0), - ("DlgFontBold8", "Tahoma", 8, None, 1), #bold - ("VerdanaBold10", "Verdana", 10, None, 1), - ("VerdanaRed9", "Verdana", 9, 255, 0), - ]) - - # UI Sequences, see "InstallUISequence Table", "Using a Sequence Table" - # Numbers indicate sequence; see sequence.py for how these action integrate - add_data(db, "InstallUISequence", - [("PrepareDlg", "Not Privileged or Windows9x or Installed", 140), - ("WhichUsersDlg", "Privileged and not Windows9x and not Installed", 141), - # In the user interface, assume all-users installation if privileged. - ("SelectFeaturesDlg", "Not Installed", 1230), - # XXX no support for resume installations yet - #("ResumeDlg", "Installed AND (RESUME OR Preselected)", 1240), - ("MaintenanceTypeDlg", "Installed AND NOT RESUME AND NOT Preselected", 1250), - ("ProgressDlg", None, 1280)]) - - add_data(db, 'ActionText', text.ActionText) - add_data(db, 'UIText', text.UIText) - ##################################################################### - # Standard dialogs: FatalError, UserExit, ExitDialog - fatal=PyDialog(db, "FatalError", x, y, w, h, modal, title, - "Finish", "Finish", "Finish") - fatal.title("[ProductName] Installer ended prematurely") - fatal.back("< Back", "Finish", active = 0) - fatal.cancel("Cancel", "Back", active = 0) - fatal.text("Description1", 15, 70, 320, 80, 0x30003, - "[ProductName] setup ended prematurely because of an error. Your system has not been modified. To install this program at a later time, please run the installation again.") - fatal.text("Description2", 15, 155, 320, 20, 0x30003, - "Click the Finish button to exit the Installer.") - c=fatal.next("Finish", "Cancel", name="Finish") - c.event("EndDialog", "Exit") - - user_exit=PyDialog(db, "UserExit", x, y, w, h, modal, title, - "Finish", "Finish", "Finish") - user_exit.title("[ProductName] Installer was interrupted") - user_exit.back("< Back", "Finish", active = 0) - user_exit.cancel("Cancel", "Back", active = 0) - user_exit.text("Description1", 15, 70, 320, 80, 0x30003, - "[ProductName] setup was interrupted. Your system has not been modified. " - "To install this program at a later time, please run the installation again.") - user_exit.text("Description2", 15, 155, 320, 20, 0x30003, - "Click the Finish button to exit the Installer.") - c = user_exit.next("Finish", "Cancel", name="Finish") - c.event("EndDialog", "Exit") - - exit_dialog = PyDialog(db, "ExitDialog", x, y, w, h, modal, title, - "Finish", "Finish", "Finish") - exit_dialog.title("Completing the [ProductName] Installer") - exit_dialog.back("< Back", "Finish", active = 0) - exit_dialog.cancel("Cancel", "Back", active = 0) - exit_dialog.text("Description", 15, 235, 320, 20, 0x30003, - "Click the Finish button to exit the Installer.") - c = exit_dialog.next("Finish", "Cancel", name="Finish") - c.event("EndDialog", "Return") - - ##################################################################### - # Required dialog: FilesInUse, ErrorDlg - inuse = PyDialog(db, "FilesInUse", - x, y, w, h, - 19, # KeepModeless|Modal|Visible - title, - "Retry", "Retry", "Retry", bitmap=False) - inuse.text("Title", 15, 6, 200, 15, 0x30003, - r"{\DlgFontBold8}Files in Use") - inuse.text("Description", 20, 23, 280, 20, 0x30003, - "Some files that need to be updated are currently in use.") - inuse.text("Text", 20, 55, 330, 50, 3, - "The following applications are using files that need to be updated by this setup. Close these applications and then click Retry to continue the installation or Cancel to exit it.") - inuse.control("List", "ListBox", 20, 107, 330, 130, 7, "FileInUseProcess", - None, None, None) - c=inuse.back("Exit", "Ignore", name="Exit") - c.event("EndDialog", "Exit") - c=inuse.next("Ignore", "Retry", name="Ignore") - c.event("EndDialog", "Ignore") - c=inuse.cancel("Retry", "Exit", name="Retry") - c.event("EndDialog","Retry") - - # See "Error Dialog". See "ICE20" for the required names of the controls. - error = Dialog(db, "ErrorDlg", - 50, 10, 330, 101, - 65543, # Error|Minimize|Modal|Visible - title, - "ErrorText", None, None) - error.text("ErrorText", 50,9,280,48,3, "") - #error.control("ErrorIcon", "Icon", 15, 9, 24, 24, 5242881, None, "py.ico", None, None) - error.pushbutton("N",120,72,81,21,3,"No",None).event("EndDialog","ErrorNo") - error.pushbutton("Y",240,72,81,21,3,"Yes",None).event("EndDialog","ErrorYes") - error.pushbutton("A",0,72,81,21,3,"Abort",None).event("EndDialog","ErrorAbort") - error.pushbutton("C",42,72,81,21,3,"Cancel",None).event("EndDialog","ErrorCancel") - error.pushbutton("I",81,72,81,21,3,"Ignore",None).event("EndDialog","ErrorIgnore") - error.pushbutton("O",159,72,81,21,3,"Ok",None).event("EndDialog","ErrorOk") - error.pushbutton("R",198,72,81,21,3,"Retry",None).event("EndDialog","ErrorRetry") - - ##################################################################### - # Global "Query Cancel" dialog - cancel = Dialog(db, "CancelDlg", 50, 10, 260, 85, 3, title, - "No", "No", "No") - cancel.text("Text", 48, 15, 194, 30, 3, - "Are you sure you want to cancel [ProductName] installation?") - #cancel.control("Icon", "Icon", 15, 15, 24, 24, 5242881, None, - # "py.ico", None, None) - c=cancel.pushbutton("Yes", 72, 57, 56, 17, 3, "Yes", "No") - c.event("EndDialog", "Exit") - - c=cancel.pushbutton("No", 132, 57, 56, 17, 3, "No", "Yes") - c.event("EndDialog", "Return") - - ##################################################################### - # Global "Wait for costing" dialog - costing = Dialog(db, "WaitForCostingDlg", 50, 10, 260, 85, modal, title, - "Return", "Return", "Return") - costing.text("Text", 48, 15, 194, 30, 3, - "Please wait while the installer finishes determining your disk space requirements.") - c = costing.pushbutton("Return", 102, 57, 56, 17, 3, "Return", None) - c.event("EndDialog", "Exit") - - ##################################################################### - # Preparation dialog: no user input except cancellation - prep = PyDialog(db, "PrepareDlg", x, y, w, h, modeless, title, - "Cancel", "Cancel", "Cancel") - prep.text("Description", 15, 70, 320, 40, 0x30003, - "Please wait while the Installer prepares to guide you through the installation.") - prep.title("Welcome to the [ProductName] Installer") - c=prep.text("ActionText", 15, 110, 320, 20, 0x30003, "Pondering...") - c.mapping("ActionText", "Text") - c=prep.text("ActionData", 15, 135, 320, 30, 0x30003, None) - c.mapping("ActionData", "Text") - prep.back("Back", None, active=0) - prep.next("Next", None, active=0) - c=prep.cancel("Cancel", None) - c.event("SpawnDialog", "CancelDlg") - - ##################################################################### - # Feature (Python directory) selection - seldlg = PyDialog(db, "SelectFeaturesDlg", x, y, w, h, modal, title, - "Next", "Next", "Cancel") - seldlg.title("Select Python Installations") - - seldlg.text("Hint", 15, 30, 300, 20, 3, - "Select the Python locations where %s should be installed." - % self.distribution.get_fullname()) - - seldlg.back("< Back", None, active=0) - c = seldlg.next("Next >", "Cancel") - order = 1 - c.event("[TARGETDIR]", "[SourceDir]", ordering=order) - for version in self.versions + [self.other_version]: - order += 1 - c.event("[TARGETDIR]", "[TARGETDIR%s]" % version, - "FEATURE_SELECTED AND &Python%s=3" % version, - ordering=order) - c.event("SpawnWaitDialog", "WaitForCostingDlg", ordering=order + 1) - c.event("EndDialog", "Return", ordering=order + 2) - c = seldlg.cancel("Cancel", "Features") - c.event("SpawnDialog", "CancelDlg") - - c = seldlg.control("Features", "SelectionTree", 15, 60, 300, 120, 3, - "FEATURE", None, "PathEdit", None) - c.event("[FEATURE_SELECTED]", "1") - ver = self.other_version - install_other_cond = "FEATURE_SELECTED AND &Python%s=3" % ver - dont_install_other_cond = "FEATURE_SELECTED AND &Python%s<>3" % ver - - c = seldlg.text("Other", 15, 200, 300, 15, 3, - "Provide an alternate Python location") - c.condition("Enable", install_other_cond) - c.condition("Show", install_other_cond) - c.condition("Disable", dont_install_other_cond) - c.condition("Hide", dont_install_other_cond) - - c = seldlg.control("PathEdit", "PathEdit", 15, 215, 300, 16, 1, - "TARGETDIR" + ver, None, "Next", None) - c.condition("Enable", install_other_cond) - c.condition("Show", install_other_cond) - c.condition("Disable", dont_install_other_cond) - c.condition("Hide", dont_install_other_cond) - - ##################################################################### - # Disk cost - cost = PyDialog(db, "DiskCostDlg", x, y, w, h, modal, title, - "OK", "OK", "OK", bitmap=False) - cost.text("Title", 15, 6, 200, 15, 0x30003, - r"{\DlgFontBold8}Disk Space Requirements") - cost.text("Description", 20, 20, 280, 20, 0x30003, - "The disk space required for the installation of the selected features.") - cost.text("Text", 20, 53, 330, 60, 3, - "The highlighted volumes (if any) do not have enough disk space " - "available for the currently selected features. You can either " - "remove some files from the highlighted volumes, or choose to " - "install less features onto local drive(s), or select different " - "destination drive(s).") - cost.control("VolumeList", "VolumeCostList", 20, 100, 330, 150, 393223, - None, "{120}{70}{70}{70}{70}", None, None) - cost.xbutton("OK", "Ok", None, 0.5).event("EndDialog", "Return") - - ##################################################################### - # WhichUsers Dialog. Only available on NT, and for privileged users. - # This must be run before FindRelatedProducts, because that will - # take into account whether the previous installation was per-user - # or per-machine. We currently don't support going back to this - # dialog after "Next" was selected; to support this, we would need to - # find how to reset the ALLUSERS property, and how to re-run - # FindRelatedProducts. - # On Windows9x, the ALLUSERS property is ignored on the command line - # and in the Property table, but installer fails according to the documentation - # if a dialog attempts to set ALLUSERS. - whichusers = PyDialog(db, "WhichUsersDlg", x, y, w, h, modal, title, - "AdminInstall", "Next", "Cancel") - whichusers.title("Select whether to install [ProductName] for all users of this computer.") - # A radio group with two options: allusers, justme - g = whichusers.radiogroup("AdminInstall", 15, 60, 260, 50, 3, - "WhichUsers", "", "Next") - g.add("ALL", 0, 5, 150, 20, "Install for all users") - g.add("JUSTME", 0, 25, 150, 20, "Install just for me") - - whichusers.back("Back", None, active=0) - - c = whichusers.next("Next >", "Cancel") - c.event("[ALLUSERS]", "1", 'WhichUsers="ALL"', 1) - c.event("EndDialog", "Return", ordering = 2) - - c = whichusers.cancel("Cancel", "AdminInstall") - c.event("SpawnDialog", "CancelDlg") - - ##################################################################### - # Installation Progress dialog (modeless) - progress = PyDialog(db, "ProgressDlg", x, y, w, h, modeless, title, - "Cancel", "Cancel", "Cancel", bitmap=False) - progress.text("Title", 20, 15, 200, 15, 0x30003, - r"{\DlgFontBold8}[Progress1] [ProductName]") - progress.text("Text", 35, 65, 300, 30, 3, - "Please wait while the Installer [Progress2] [ProductName]. " - "This may take several minutes.") - progress.text("StatusLabel", 35, 100, 35, 20, 3, "Status:") - - c=progress.text("ActionText", 70, 100, w-70, 20, 3, "Pondering...") - c.mapping("ActionText", "Text") - - #c=progress.text("ActionData", 35, 140, 300, 20, 3, None) - #c.mapping("ActionData", "Text") - - c=progress.control("ProgressBar", "ProgressBar", 35, 120, 300, 10, 65537, - None, "Progress done", None, None) - c.mapping("SetProgress", "Progress") - - progress.back("< Back", "Next", active=False) - progress.next("Next >", "Cancel", active=False) - progress.cancel("Cancel", "Back").event("SpawnDialog", "CancelDlg") - - ################################################################### - # Maintenance type: repair/uninstall - maint = PyDialog(db, "MaintenanceTypeDlg", x, y, w, h, modal, title, - "Next", "Next", "Cancel") - maint.title("Welcome to the [ProductName] Setup Wizard") - maint.text("BodyText", 15, 63, 330, 42, 3, - "Select whether you want to repair or remove [ProductName].") - g=maint.radiogroup("RepairRadioGroup", 15, 108, 330, 60, 3, - "MaintenanceForm_Action", "", "Next") - #g.add("Change", 0, 0, 200, 17, "&Change [ProductName]") - g.add("Repair", 0, 18, 200, 17, "&Repair [ProductName]") - g.add("Remove", 0, 36, 200, 17, "Re&move [ProductName]") - - maint.back("< Back", None, active=False) - c=maint.next("Finish", "Cancel") - # Change installation: Change progress dialog to "Change", then ask - # for feature selection - #c.event("[Progress1]", "Change", 'MaintenanceForm_Action="Change"', 1) - #c.event("[Progress2]", "changes", 'MaintenanceForm_Action="Change"', 2) - - # Reinstall: Change progress dialog to "Repair", then invoke reinstall - # Also set list of reinstalled features to "ALL" - c.event("[REINSTALL]", "ALL", 'MaintenanceForm_Action="Repair"', 5) - c.event("[Progress1]", "Repairing", 'MaintenanceForm_Action="Repair"', 6) - c.event("[Progress2]", "repairs", 'MaintenanceForm_Action="Repair"', 7) - c.event("Reinstall", "ALL", 'MaintenanceForm_Action="Repair"', 8) - - # Uninstall: Change progress to "Remove", then invoke uninstall - # Also set list of removed features to "ALL" - c.event("[REMOVE]", "ALL", 'MaintenanceForm_Action="Remove"', 11) - c.event("[Progress1]", "Removing", 'MaintenanceForm_Action="Remove"', 12) - c.event("[Progress2]", "removes", 'MaintenanceForm_Action="Remove"', 13) - c.event("Remove", "ALL", 'MaintenanceForm_Action="Remove"', 14) - - # Close dialog when maintenance action scheduled - c.event("EndDialog", "Return", 'MaintenanceForm_Action<>"Change"', 20) - #c.event("NewDialog", "SelectFeaturesDlg", 'MaintenanceForm_Action="Change"', 21) - - maint.cancel("Cancel", "RepairRadioGroup").event("SpawnDialog", "CancelDlg") - - def get_installer_filename(self, fullname): - # Factored out to allow overriding in subclasses - if self.target_version: - base_name = "%s.%s-py%s.msi" % (fullname, self.plat_name, - self.target_version) - else: - base_name = "%s.%s.msi" % (fullname, self.plat_name) - installer_name = os.path.join(self.dist_dir, base_name) - return installer_name diff --git a/Lib/distutils/command/bdist_rpm.py b/Lib/distutils/command/bdist_rpm.py deleted file mode 100644 index 02f10dd89d9..00000000000 --- a/Lib/distutils/command/bdist_rpm.py +++ /dev/null @@ -1,582 +0,0 @@ -"""distutils.command.bdist_rpm - -Implements the Distutils 'bdist_rpm' command (create RPM source and binary -distributions).""" - -import subprocess, sys, os -from distutils.core import Command -from distutils.debug import DEBUG -from distutils.util import get_platform -from distutils.file_util import write_file -from distutils.errors import * -from distutils.sysconfig import get_python_version -from distutils import log - -class bdist_rpm(Command): - - description = "create an RPM distribution" - - user_options = [ - ('bdist-base=', None, - "base directory for creating built distributions"), - ('rpm-base=', None, - "base directory for creating RPMs (defaults to \"rpm\" under " - "--bdist-base; must be specified for RPM 2)"), - ('dist-dir=', 'd', - "directory to put final RPM files in " - "(and .spec files if --spec-only)"), - ('python=', None, - "path to Python interpreter to hard-code in the .spec file " - "(default: \"python\")"), - ('fix-python', None, - "hard-code the exact path to the current Python interpreter in " - "the .spec file"), - ('spec-only', None, - "only regenerate spec file"), - ('source-only', None, - "only generate source RPM"), - ('binary-only', None, - "only generate binary RPM"), - ('use-bzip2', None, - "use bzip2 instead of gzip to create source distribution"), - - # More meta-data: too RPM-specific to put in the setup script, - # but needs to go in the .spec file -- so we make these options - # to "bdist_rpm". The idea is that packagers would put this - # info in setup.cfg, although they are of course free to - # supply it on the command line. - ('distribution-name=', None, - "name of the (Linux) distribution to which this " - "RPM applies (*not* the name of the module distribution!)"), - ('group=', None, - "package classification [default: \"Development/Libraries\"]"), - ('release=', None, - "RPM release number"), - ('serial=', None, - "RPM serial number"), - ('vendor=', None, - "RPM \"vendor\" (eg. \"Joe Blow \") " - "[default: maintainer or author from setup script]"), - ('packager=', None, - "RPM packager (eg. \"Jane Doe \") " - "[default: vendor]"), - ('doc-files=', None, - "list of documentation files (space or comma-separated)"), - ('changelog=', None, - "RPM changelog"), - ('icon=', None, - "name of icon file"), - ('provides=', None, - "capabilities provided by this package"), - ('requires=', None, - "capabilities required by this package"), - ('conflicts=', None, - "capabilities which conflict with this package"), - ('build-requires=', None, - "capabilities required to build this package"), - ('obsoletes=', None, - "capabilities made obsolete by this package"), - ('no-autoreq', None, - "do not automatically calculate dependencies"), - - # Actions to take when building RPM - ('keep-temp', 'k', - "don't clean up RPM build directory"), - ('no-keep-temp', None, - "clean up RPM build directory [default]"), - ('use-rpm-opt-flags', None, - "compile with RPM_OPT_FLAGS when building from source RPM"), - ('no-rpm-opt-flags', None, - "do not pass any RPM CFLAGS to compiler"), - ('rpm3-mode', None, - "RPM 3 compatibility mode (default)"), - ('rpm2-mode', None, - "RPM 2 compatibility mode"), - - # Add the hooks necessary for specifying custom scripts - ('prep-script=', None, - "Specify a script for the PREP phase of RPM building"), - ('build-script=', None, - "Specify a script for the BUILD phase of RPM building"), - - ('pre-install=', None, - "Specify a script for the pre-INSTALL phase of RPM building"), - ('install-script=', None, - "Specify a script for the INSTALL phase of RPM building"), - ('post-install=', None, - "Specify a script for the post-INSTALL phase of RPM building"), - - ('pre-uninstall=', None, - "Specify a script for the pre-UNINSTALL phase of RPM building"), - ('post-uninstall=', None, - "Specify a script for the post-UNINSTALL phase of RPM building"), - - ('clean-script=', None, - "Specify a script for the CLEAN phase of RPM building"), - - ('verify-script=', None, - "Specify a script for the VERIFY phase of the RPM build"), - - # Allow a packager to explicitly force an architecture - ('force-arch=', None, - "Force an architecture onto the RPM build process"), - - ('quiet', 'q', - "Run the INSTALL phase of RPM building in quiet mode"), - ] - - boolean_options = ['keep-temp', 'use-rpm-opt-flags', 'rpm3-mode', - 'no-autoreq', 'quiet'] - - negative_opt = {'no-keep-temp': 'keep-temp', - 'no-rpm-opt-flags': 'use-rpm-opt-flags', - 'rpm2-mode': 'rpm3-mode'} - - - def initialize_options(self): - self.bdist_base = None - self.rpm_base = None - self.dist_dir = None - self.python = None - self.fix_python = None - self.spec_only = None - self.binary_only = None - self.source_only = None - self.use_bzip2 = None - - self.distribution_name = None - self.group = None - self.release = None - self.serial = None - self.vendor = None - self.packager = None - self.doc_files = None - self.changelog = None - self.icon = None - - self.prep_script = None - self.build_script = None - self.install_script = None - self.clean_script = None - self.verify_script = None - self.pre_install = None - self.post_install = None - self.pre_uninstall = None - self.post_uninstall = None - self.prep = None - self.provides = None - self.requires = None - self.conflicts = None - self.build_requires = None - self.obsoletes = None - - self.keep_temp = 0 - self.use_rpm_opt_flags = 1 - self.rpm3_mode = 1 - self.no_autoreq = 0 - - self.force_arch = None - self.quiet = 0 - - def finalize_options(self): - self.set_undefined_options('bdist', ('bdist_base', 'bdist_base')) - if self.rpm_base is None: - if not self.rpm3_mode: - raise DistutilsOptionError( - "you must specify --rpm-base in RPM 2 mode") - self.rpm_base = os.path.join(self.bdist_base, "rpm") - - if self.python is None: - if self.fix_python: - self.python = sys.executable - else: - self.python = "python3" - elif self.fix_python: - raise DistutilsOptionError( - "--python and --fix-python are mutually exclusive options") - - if os.name != 'posix': - raise DistutilsPlatformError("don't know how to create RPM " - "distributions on platform %s" % os.name) - if self.binary_only and self.source_only: - raise DistutilsOptionError( - "cannot supply both '--source-only' and '--binary-only'") - - # don't pass CFLAGS to pure python distributions - if not self.distribution.has_ext_modules(): - self.use_rpm_opt_flags = 0 - - self.set_undefined_options('bdist', ('dist_dir', 'dist_dir')) - self.finalize_package_data() - - def finalize_package_data(self): - self.ensure_string('group', "Development/Libraries") - self.ensure_string('vendor', - "%s <%s>" % (self.distribution.get_contact(), - self.distribution.get_contact_email())) - self.ensure_string('packager') - self.ensure_string_list('doc_files') - if isinstance(self.doc_files, list): - for readme in ('README', 'README.txt'): - if os.path.exists(readme) and readme not in self.doc_files: - self.doc_files.append(readme) - - self.ensure_string('release', "1") - self.ensure_string('serial') # should it be an int? - - self.ensure_string('distribution_name') - - self.ensure_string('changelog') - # Format changelog correctly - self.changelog = self._format_changelog(self.changelog) - - self.ensure_filename('icon') - - self.ensure_filename('prep_script') - self.ensure_filename('build_script') - self.ensure_filename('install_script') - self.ensure_filename('clean_script') - self.ensure_filename('verify_script') - self.ensure_filename('pre_install') - self.ensure_filename('post_install') - self.ensure_filename('pre_uninstall') - self.ensure_filename('post_uninstall') - - # XXX don't forget we punted on summaries and descriptions -- they - # should be handled here eventually! - - # Now *this* is some meta-data that belongs in the setup script... - self.ensure_string_list('provides') - self.ensure_string_list('requires') - self.ensure_string_list('conflicts') - self.ensure_string_list('build_requires') - self.ensure_string_list('obsoletes') - - self.ensure_string('force_arch') - - def run(self): - if DEBUG: - print("before _get_package_data():") - print("vendor =", self.vendor) - print("packager =", self.packager) - print("doc_files =", self.doc_files) - print("changelog =", self.changelog) - - # make directories - if self.spec_only: - spec_dir = self.dist_dir - self.mkpath(spec_dir) - else: - rpm_dir = {} - for d in ('SOURCES', 'SPECS', 'BUILD', 'RPMS', 'SRPMS'): - rpm_dir[d] = os.path.join(self.rpm_base, d) - self.mkpath(rpm_dir[d]) - spec_dir = rpm_dir['SPECS'] - - # Spec file goes into 'dist_dir' if '--spec-only specified', - # build/rpm. otherwise. - spec_path = os.path.join(spec_dir, - "%s.spec" % self.distribution.get_name()) - self.execute(write_file, - (spec_path, - self._make_spec_file()), - "writing '%s'" % spec_path) - - if self.spec_only: # stop if requested - return - - # Make a source distribution and copy to SOURCES directory with - # optional icon. - saved_dist_files = self.distribution.dist_files[:] - sdist = self.reinitialize_command('sdist') - if self.use_bzip2: - sdist.formats = ['bztar'] - else: - sdist.formats = ['gztar'] - self.run_command('sdist') - self.distribution.dist_files = saved_dist_files - - source = sdist.get_archive_files()[0] - source_dir = rpm_dir['SOURCES'] - self.copy_file(source, source_dir) - - if self.icon: - if os.path.exists(self.icon): - self.copy_file(self.icon, source_dir) - else: - raise DistutilsFileError( - "icon file '%s' does not exist" % self.icon) - - # build package - log.info("building RPMs") - rpm_cmd = ['rpm'] - if os.path.exists('/usr/bin/rpmbuild') or \ - os.path.exists('/bin/rpmbuild'): - rpm_cmd = ['rpmbuild'] - - if self.source_only: # what kind of RPMs? - rpm_cmd.append('-bs') - elif self.binary_only: - rpm_cmd.append('-bb') - else: - rpm_cmd.append('-ba') - rpm_cmd.extend(['--define', '__python %s' % self.python]) - if self.rpm3_mode: - rpm_cmd.extend(['--define', - '_topdir %s' % os.path.abspath(self.rpm_base)]) - if not self.keep_temp: - rpm_cmd.append('--clean') - - if self.quiet: - rpm_cmd.append('--quiet') - - rpm_cmd.append(spec_path) - # Determine the binary rpm names that should be built out of this spec - # file - # Note that some of these may not be really built (if the file - # list is empty) - nvr_string = "%{name}-%{version}-%{release}" - src_rpm = nvr_string + ".src.rpm" - non_src_rpm = "%{arch}/" + nvr_string + ".%{arch}.rpm" - q_cmd = r"rpm -q --qf '%s %s\n' --specfile '%s'" % ( - src_rpm, non_src_rpm, spec_path) - - out = os.popen(q_cmd) - try: - binary_rpms = [] - source_rpm = None - while True: - line = out.readline() - if not line: - break - l = line.strip().split() - assert(len(l) == 2) - binary_rpms.append(l[1]) - # The source rpm is named after the first entry in the spec file - if source_rpm is None: - source_rpm = l[0] - - status = out.close() - if status: - raise DistutilsExecError("Failed to execute: %s" % repr(q_cmd)) - - finally: - out.close() - - self.spawn(rpm_cmd) - - if not self.dry_run: - if self.distribution.has_ext_modules(): - pyversion = get_python_version() - else: - pyversion = 'any' - - if not self.binary_only: - srpm = os.path.join(rpm_dir['SRPMS'], source_rpm) - assert(os.path.exists(srpm)) - self.move_file(srpm, self.dist_dir) - filename = os.path.join(self.dist_dir, source_rpm) - self.distribution.dist_files.append( - ('bdist_rpm', pyversion, filename)) - - if not self.source_only: - for rpm in binary_rpms: - rpm = os.path.join(rpm_dir['RPMS'], rpm) - if os.path.exists(rpm): - self.move_file(rpm, self.dist_dir) - filename = os.path.join(self.dist_dir, - os.path.basename(rpm)) - self.distribution.dist_files.append( - ('bdist_rpm', pyversion, filename)) - - def _dist_path(self, path): - return os.path.join(self.dist_dir, os.path.basename(path)) - - def _make_spec_file(self): - """Generate the text of an RPM spec file and return it as a - list of strings (one per line). - """ - # definitions and headers - spec_file = [ - '%define name ' + self.distribution.get_name(), - '%define version ' + self.distribution.get_version().replace('-','_'), - '%define unmangled_version ' + self.distribution.get_version(), - '%define release ' + self.release.replace('-','_'), - '', - 'Summary: ' + self.distribution.get_description(), - ] - - # Workaround for #14443 which affects some RPM based systems such as - # RHEL6 (and probably derivatives) - vendor_hook = subprocess.getoutput('rpm --eval %{__os_install_post}') - # Generate a potential replacement value for __os_install_post (whilst - # normalizing the whitespace to simplify the test for whether the - # invocation of brp-python-bytecompile passes in __python): - vendor_hook = '\n'.join([' %s \\' % line.strip() - for line in vendor_hook.splitlines()]) - problem = "brp-python-bytecompile \\\n" - fixed = "brp-python-bytecompile %{__python} \\\n" - fixed_hook = vendor_hook.replace(problem, fixed) - if fixed_hook != vendor_hook: - spec_file.append('# Workaround for http://bugs.python.org/issue14443') - spec_file.append('%define __os_install_post ' + fixed_hook + '\n') - - # put locale summaries into spec file - # XXX not supported for now (hard to put a dictionary - # in a config file -- arg!) - #for locale in self.summaries.keys(): - # spec_file.append('Summary(%s): %s' % (locale, - # self.summaries[locale])) - - spec_file.extend([ - 'Name: %{name}', - 'Version: %{version}', - 'Release: %{release}',]) - - # XXX yuck! this filename is available from the "sdist" command, - # but only after it has run: and we create the spec file before - # running "sdist", in case of --spec-only. - if self.use_bzip2: - spec_file.append('Source0: %{name}-%{unmangled_version}.tar.bz2') - else: - spec_file.append('Source0: %{name}-%{unmangled_version}.tar.gz') - - spec_file.extend([ - 'License: ' + self.distribution.get_license(), - 'Group: ' + self.group, - 'BuildRoot: %{_tmppath}/%{name}-%{version}-%{release}-buildroot', - 'Prefix: %{_prefix}', ]) - - if not self.force_arch: - # noarch if no extension modules - if not self.distribution.has_ext_modules(): - spec_file.append('BuildArch: noarch') - else: - spec_file.append( 'BuildArch: %s' % self.force_arch ) - - for field in ('Vendor', - 'Packager', - 'Provides', - 'Requires', - 'Conflicts', - 'Obsoletes', - ): - val = getattr(self, field.lower()) - if isinstance(val, list): - spec_file.append('%s: %s' % (field, ' '.join(val))) - elif val is not None: - spec_file.append('%s: %s' % (field, val)) - - - if self.distribution.get_url() != 'UNKNOWN': - spec_file.append('Url: ' + self.distribution.get_url()) - - if self.distribution_name: - spec_file.append('Distribution: ' + self.distribution_name) - - if self.build_requires: - spec_file.append('BuildRequires: ' + - ' '.join(self.build_requires)) - - if self.icon: - spec_file.append('Icon: ' + os.path.basename(self.icon)) - - if self.no_autoreq: - spec_file.append('AutoReq: 0') - - spec_file.extend([ - '', - '%description', - self.distribution.get_long_description() - ]) - - # put locale descriptions into spec file - # XXX again, suppressed because config file syntax doesn't - # easily support this ;-( - #for locale in self.descriptions.keys(): - # spec_file.extend([ - # '', - # '%description -l ' + locale, - # self.descriptions[locale], - # ]) - - # rpm scripts - # figure out default build script - def_setup_call = "%s %s" % (self.python,os.path.basename(sys.argv[0])) - def_build = "%s build" % def_setup_call - if self.use_rpm_opt_flags: - def_build = 'env CFLAGS="$RPM_OPT_FLAGS" ' + def_build - - # insert contents of files - - # XXX this is kind of misleading: user-supplied options are files - # that we open and interpolate into the spec file, but the defaults - # are just text that we drop in as-is. Hmmm. - - install_cmd = ('%s install -O1 --root=$RPM_BUILD_ROOT ' - '--record=INSTALLED_FILES') % def_setup_call - - script_options = [ - ('prep', 'prep_script', "%setup -n %{name}-%{unmangled_version}"), - ('build', 'build_script', def_build), - ('install', 'install_script', install_cmd), - ('clean', 'clean_script', "rm -rf $RPM_BUILD_ROOT"), - ('verifyscript', 'verify_script', None), - ('pre', 'pre_install', None), - ('post', 'post_install', None), - ('preun', 'pre_uninstall', None), - ('postun', 'post_uninstall', None), - ] - - for (rpm_opt, attr, default) in script_options: - # Insert contents of file referred to, if no file is referred to - # use 'default' as contents of script - val = getattr(self, attr) - if val or default: - spec_file.extend([ - '', - '%' + rpm_opt,]) - if val: - spec_file.extend(open(val, 'r').read().split('\n')) - else: - spec_file.append(default) - - - # files section - spec_file.extend([ - '', - '%files -f INSTALLED_FILES', - '%defattr(-,root,root)', - ]) - - if self.doc_files: - spec_file.append('%doc ' + ' '.join(self.doc_files)) - - if self.changelog: - spec_file.extend([ - '', - '%changelog',]) - spec_file.extend(self.changelog) - - return spec_file - - def _format_changelog(self, changelog): - """Format the changelog correctly and convert it to a list of strings - """ - if not changelog: - return changelog - new_changelog = [] - for line in changelog.strip().split('\n'): - line = line.strip() - if line[0] == '*': - new_changelog.extend(['', line]) - elif line[0] == '-': - new_changelog.append(line) - else: - new_changelog.append(' ' + line) - - # strip trailing newline inserted by first changelog entry - if not new_changelog[0]: - del new_changelog[0] - - return new_changelog diff --git a/Lib/distutils/command/bdist_wininst.py b/Lib/distutils/command/bdist_wininst.py deleted file mode 100644 index 1db47f9b983..00000000000 --- a/Lib/distutils/command/bdist_wininst.py +++ /dev/null @@ -1,367 +0,0 @@ -"""distutils.command.bdist_wininst - -Implements the Distutils 'bdist_wininst' command: create a windows installer -exe-program.""" - -import sys, os -from distutils.core import Command -from distutils.util import get_platform -from distutils.dir_util import create_tree, remove_tree -from distutils.errors import * -from distutils.sysconfig import get_python_version -from distutils import log - -class bdist_wininst(Command): - - description = "create an executable installer for MS Windows" - - user_options = [('bdist-dir=', None, - "temporary directory for creating the distribution"), - ('plat-name=', 'p', - "platform name to embed in generated filenames " - "(default: %s)" % get_platform()), - ('keep-temp', 'k', - "keep the pseudo-installation tree around after " + - "creating the distribution archive"), - ('target-version=', None, - "require a specific python version" + - " on the target system"), - ('no-target-compile', 'c', - "do not compile .py to .pyc on the target system"), - ('no-target-optimize', 'o', - "do not compile .py to .pyo (optimized) " - "on the target system"), - ('dist-dir=', 'd', - "directory to put final built distributions in"), - ('bitmap=', 'b', - "bitmap to use for the installer instead of python-powered logo"), - ('title=', 't', - "title to display on the installer background instead of default"), - ('skip-build', None, - "skip rebuilding everything (for testing/debugging)"), - ('install-script=', None, - "basename of installation script to be run after " - "installation or before deinstallation"), - ('pre-install-script=', None, - "Fully qualified filename of a script to be run before " - "any files are installed. This script need not be in the " - "distribution"), - ('user-access-control=', None, - "specify Vista's UAC handling - 'none'/default=no " - "handling, 'auto'=use UAC if target Python installed for " - "all users, 'force'=always use UAC"), - ] - - boolean_options = ['keep-temp', 'no-target-compile', 'no-target-optimize', - 'skip-build'] - - def initialize_options(self): - self.bdist_dir = None - self.plat_name = None - self.keep_temp = 0 - self.no_target_compile = 0 - self.no_target_optimize = 0 - self.target_version = None - self.dist_dir = None - self.bitmap = None - self.title = None - self.skip_build = None - self.install_script = None - self.pre_install_script = None - self.user_access_control = None - - - def finalize_options(self): - self.set_undefined_options('bdist', ('skip_build', 'skip_build')) - - if self.bdist_dir is None: - if self.skip_build and self.plat_name: - # If build is skipped and plat_name is overridden, bdist will - # not see the correct 'plat_name' - so set that up manually. - bdist = self.distribution.get_command_obj('bdist') - bdist.plat_name = self.plat_name - # next the command will be initialized using that name - bdist_base = self.get_finalized_command('bdist').bdist_base - self.bdist_dir = os.path.join(bdist_base, 'wininst') - - if not self.target_version: - self.target_version = "" - - if not self.skip_build and self.distribution.has_ext_modules(): - short_version = get_python_version() - if self.target_version and self.target_version != short_version: - raise DistutilsOptionError( - "target version can only be %s, or the '--skip-build'" \ - " option must be specified" % (short_version,)) - self.target_version = short_version - - self.set_undefined_options('bdist', - ('dist_dir', 'dist_dir'), - ('plat_name', 'plat_name'), - ) - - if self.install_script: - for script in self.distribution.scripts: - if self.install_script == os.path.basename(script): - break - else: - raise DistutilsOptionError( - "install_script '%s' not found in scripts" - % self.install_script) - - def run(self): - if (sys.platform != "win32" and - (self.distribution.has_ext_modules() or - self.distribution.has_c_libraries())): - raise DistutilsPlatformError \ - ("distribution contains extensions and/or C libraries; " - "must be compiled on a Windows 32 platform") - - if not self.skip_build: - self.run_command('build') - - install = self.reinitialize_command('install', reinit_subcommands=1) - install.root = self.bdist_dir - install.skip_build = self.skip_build - install.warn_dir = 0 - install.plat_name = self.plat_name - - install_lib = self.reinitialize_command('install_lib') - # we do not want to include pyc or pyo files - install_lib.compile = 0 - install_lib.optimize = 0 - - if self.distribution.has_ext_modules(): - # If we are building an installer for a Python version other - # than the one we are currently running, then we need to ensure - # our build_lib reflects the other Python version rather than ours. - # Note that for target_version!=sys.version, we must have skipped the - # build step, so there is no issue with enforcing the build of this - # version. - target_version = self.target_version - if not target_version: - assert self.skip_build, "Should have already checked this" - target_version = '%d.%d' % sys.version_info[:2] - plat_specifier = ".%s-%s" % (self.plat_name, target_version) - build = self.get_finalized_command('build') - build.build_lib = os.path.join(build.build_base, - 'lib' + plat_specifier) - - # Use a custom scheme for the zip-file, because we have to decide - # at installation time which scheme to use. - for key in ('purelib', 'platlib', 'headers', 'scripts', 'data'): - value = key.upper() - if key == 'headers': - value = value + '/Include/$dist_name' - setattr(install, - 'install_' + key, - value) - - log.info("installing to %s", self.bdist_dir) - install.ensure_finalized() - - # avoid warning of 'install_lib' about installing - # into a directory not in sys.path - sys.path.insert(0, os.path.join(self.bdist_dir, 'PURELIB')) - - install.run() - - del sys.path[0] - - # And make an archive relative to the root of the - # pseudo-installation tree. - from tempfile import mktemp - archive_basename = mktemp() - fullname = self.distribution.get_fullname() - arcname = self.make_archive(archive_basename, "zip", - root_dir=self.bdist_dir) - # create an exe containing the zip-file - self.create_exe(arcname, fullname, self.bitmap) - if self.distribution.has_ext_modules(): - pyversion = get_python_version() - else: - pyversion = 'any' - self.distribution.dist_files.append(('bdist_wininst', pyversion, - self.get_installer_filename(fullname))) - # remove the zip-file again - log.debug("removing temporary file '%s'", arcname) - os.remove(arcname) - - if not self.keep_temp: - remove_tree(self.bdist_dir, dry_run=self.dry_run) - - def get_inidata(self): - # Return data describing the installation. - lines = [] - metadata = self.distribution.metadata - - # Write the [metadata] section. - lines.append("[metadata]") - - # 'info' will be displayed in the installer's dialog box, - # describing the items to be installed. - info = (metadata.long_description or '') + '\n' - - # Escape newline characters - def escape(s): - return s.replace("\n", "\\n") - - for name in ["author", "author_email", "description", "maintainer", - "maintainer_email", "name", "url", "version"]: - data = getattr(metadata, name, "") - if data: - info = info + ("\n %s: %s" % \ - (name.capitalize(), escape(data))) - lines.append("%s=%s" % (name, escape(data))) - - # The [setup] section contains entries controlling - # the installer runtime. - lines.append("\n[Setup]") - if self.install_script: - lines.append("install_script=%s" % self.install_script) - lines.append("info=%s" % escape(info)) - lines.append("target_compile=%d" % (not self.no_target_compile)) - lines.append("target_optimize=%d" % (not self.no_target_optimize)) - if self.target_version: - lines.append("target_version=%s" % self.target_version) - if self.user_access_control: - lines.append("user_access_control=%s" % self.user_access_control) - - title = self.title or self.distribution.get_fullname() - lines.append("title=%s" % escape(title)) - import time - import distutils - build_info = "Built %s with distutils-%s" % \ - (time.ctime(time.time()), distutils.__version__) - lines.append("build_info=%s" % build_info) - return "\n".join(lines) - - def create_exe(self, arcname, fullname, bitmap=None): - import struct - - self.mkpath(self.dist_dir) - - cfgdata = self.get_inidata() - - installer_name = self.get_installer_filename(fullname) - self.announce("creating %s" % installer_name) - - if bitmap: - bitmapdata = open(bitmap, "rb").read() - bitmaplen = len(bitmapdata) - else: - bitmaplen = 0 - - file = open(installer_name, "wb") - file.write(self.get_exe_bytes()) - if bitmap: - file.write(bitmapdata) - - # Convert cfgdata from unicode to ascii, mbcs encoded - if isinstance(cfgdata, str): - cfgdata = cfgdata.encode("mbcs") - - # Append the pre-install script - cfgdata = cfgdata + b"\0" - if self.pre_install_script: - # We need to normalize newlines, so we open in text mode and - # convert back to bytes. "latin-1" simply avoids any possible - # failures. - with open(self.pre_install_script, "r", - encoding="latin-1") as script: - script_data = script.read().encode("latin-1") - cfgdata = cfgdata + script_data + b"\n\0" - else: - # empty pre-install script - cfgdata = cfgdata + b"\0" - file.write(cfgdata) - - # The 'magic number' 0x1234567B is used to make sure that the - # binary layout of 'cfgdata' is what the wininst.exe binary - # expects. If the layout changes, increment that number, make - # the corresponding changes to the wininst.exe sources, and - # recompile them. - header = struct.pack("' under the base build directory. We only use one of - # them for a given distribution, though -- - if self.build_purelib is None: - self.build_purelib = os.path.join(self.build_base, 'lib') - if self.build_platlib is None: - self.build_platlib = os.path.join(self.build_base, - 'lib' + plat_specifier) - - # 'build_lib' is the actual directory that we will use for this - # particular module distribution -- if user didn't supply it, pick - # one of 'build_purelib' or 'build_platlib'. - if self.build_lib is None: - if self.distribution.ext_modules: - self.build_lib = self.build_platlib - else: - self.build_lib = self.build_purelib - - # 'build_temp' -- temporary directory for compiler turds, - # "build/temp." - if self.build_temp is None: - self.build_temp = os.path.join(self.build_base, - 'temp' + plat_specifier) - if self.build_scripts is None: - self.build_scripts = os.path.join(self.build_base, - 'scripts-%d.%d' % sys.version_info[:2]) - - if self.executable is None: - self.executable = os.path.normpath(sys.executable) - - if isinstance(self.parallel, str): - try: - self.parallel = int(self.parallel) - except ValueError: - raise DistutilsOptionError("parallel should be an integer") - - def run(self): - # Run all relevant sub-commands. This will be some subset of: - # - build_py - pure Python modules - # - build_clib - standalone C libraries - # - build_ext - Python extensions - # - build_scripts - (Python) scripts - for cmd_name in self.get_sub_commands(): - self.run_command(cmd_name) - - - # -- Predicates for the sub-command list --------------------------- - - def has_pure_modules(self): - return self.distribution.has_pure_modules() - - def has_c_libraries(self): - return self.distribution.has_c_libraries() - - def has_ext_modules(self): - return self.distribution.has_ext_modules() - - def has_scripts(self): - return self.distribution.has_scripts() - - - sub_commands = [('build_py', has_pure_modules), - ('build_clib', has_c_libraries), - ('build_ext', has_ext_modules), - ('build_scripts', has_scripts), - ] diff --git a/Lib/distutils/command/build_clib.py b/Lib/distutils/command/build_clib.py deleted file mode 100644 index 3e20ef23cd8..00000000000 --- a/Lib/distutils/command/build_clib.py +++ /dev/null @@ -1,209 +0,0 @@ -"""distutils.command.build_clib - -Implements the Distutils 'build_clib' command, to build a C/C++ library -that is included in the module distribution and needed by an extension -module.""" - - -# XXX this module has *lots* of code ripped-off quite transparently from -# build_ext.py -- not surprisingly really, as the work required to build -# a static library from a collection of C source files is not really all -# that different from what's required to build a shared object file from -# a collection of C source files. Nevertheless, I haven't done the -# necessary refactoring to account for the overlap in code between the -# two modules, mainly because a number of subtle details changed in the -# cut 'n paste. Sigh. - -import os -from distutils.core import Command -from distutils.errors import * -from distutils.sysconfig import customize_compiler -from distutils import log - -def show_compilers(): - from distutils.ccompiler import show_compilers - show_compilers() - - -class build_clib(Command): - - description = "build C/C++ libraries used by Python extensions" - - user_options = [ - ('build-clib=', 'b', - "directory to build C/C++ libraries to"), - ('build-temp=', 't', - "directory to put temporary build by-products"), - ('debug', 'g', - "compile with debugging information"), - ('force', 'f', - "forcibly build everything (ignore file timestamps)"), - ('compiler=', 'c', - "specify the compiler type"), - ] - - boolean_options = ['debug', 'force'] - - help_options = [ - ('help-compiler', None, - "list available compilers", show_compilers), - ] - - def initialize_options(self): - self.build_clib = None - self.build_temp = None - - # List of libraries to build - self.libraries = None - - # Compilation options for all libraries - self.include_dirs = None - self.define = None - self.undef = None - self.debug = None - self.force = 0 - self.compiler = None - - - def finalize_options(self): - # This might be confusing: both build-clib and build-temp default - # to build-temp as defined by the "build" command. This is because - # I think that C libraries are really just temporary build - # by-products, at least from the point of view of building Python - # extensions -- but I want to keep my options open. - self.set_undefined_options('build', - ('build_temp', 'build_clib'), - ('build_temp', 'build_temp'), - ('compiler', 'compiler'), - ('debug', 'debug'), - ('force', 'force')) - - self.libraries = self.distribution.libraries - if self.libraries: - self.check_library_list(self.libraries) - - if self.include_dirs is None: - self.include_dirs = self.distribution.include_dirs or [] - if isinstance(self.include_dirs, str): - self.include_dirs = self.include_dirs.split(os.pathsep) - - # XXX same as for build_ext -- what about 'self.define' and - # 'self.undef' ? - - - def run(self): - if not self.libraries: - return - - # Yech -- this is cut 'n pasted from build_ext.py! - from distutils.ccompiler import new_compiler - self.compiler = new_compiler(compiler=self.compiler, - dry_run=self.dry_run, - force=self.force) - customize_compiler(self.compiler) - - if self.include_dirs is not None: - self.compiler.set_include_dirs(self.include_dirs) - if self.define is not None: - # 'define' option is a list of (name,value) tuples - for (name,value) in self.define: - self.compiler.define_macro(name, value) - if self.undef is not None: - for macro in self.undef: - self.compiler.undefine_macro(macro) - - self.build_libraries(self.libraries) - - - def check_library_list(self, libraries): - """Ensure that the list of libraries is valid. - - `library` is presumably provided as a command option 'libraries'. - This method checks that it is a list of 2-tuples, where the tuples - are (library_name, build_info_dict). - - Raise DistutilsSetupError if the structure is invalid anywhere; - just returns otherwise. - """ - if not isinstance(libraries, list): - raise DistutilsSetupError( - "'libraries' option must be a list of tuples") - - for lib in libraries: - if not isinstance(lib, tuple) and len(lib) != 2: - raise DistutilsSetupError( - "each element of 'libraries' must a 2-tuple") - - name, build_info = lib - - if not isinstance(name, str): - raise DistutilsSetupError( - "first element of each tuple in 'libraries' " - "must be a string (the library name)") - - if '/' in name or (os.sep != '/' and os.sep in name): - raise DistutilsSetupError("bad library name '%s': " - "may not contain directory separators" % lib[0]) - - if not isinstance(build_info, dict): - raise DistutilsSetupError( - "second element of each tuple in 'libraries' " - "must be a dictionary (build info)") - - - def get_library_names(self): - # Assume the library list is valid -- 'check_library_list()' is - # called from 'finalize_options()', so it should be! - if not self.libraries: - return None - - lib_names = [] - for (lib_name, build_info) in self.libraries: - lib_names.append(lib_name) - return lib_names - - - def get_source_files(self): - self.check_library_list(self.libraries) - filenames = [] - for (lib_name, build_info) in self.libraries: - sources = build_info.get('sources') - if sources is None or not isinstance(sources, (list, tuple)): - raise DistutilsSetupError( - "in 'libraries' option (library '%s'), " - "'sources' must be present and must be " - "a list of source filenames" % lib_name) - - filenames.extend(sources) - return filenames - - - def build_libraries(self, libraries): - for (lib_name, build_info) in libraries: - sources = build_info.get('sources') - if sources is None or not isinstance(sources, (list, tuple)): - raise DistutilsSetupError( - "in 'libraries' option (library '%s'), " - "'sources' must be present and must be " - "a list of source filenames" % lib_name) - sources = list(sources) - - log.info("building '%s' library", lib_name) - - # First, compile the source code to object files in the library - # directory. (This should probably change to putting object - # files in a temporary build directory.) - macros = build_info.get('macros') - include_dirs = build_info.get('include_dirs') - objects = self.compiler.compile(sources, - output_dir=self.build_temp, - macros=macros, - include_dirs=include_dirs, - debug=self.debug) - - # Now "link" the object files together into a static library. - # (On Unix at least, this isn't really linking -- it just - # builds an archive. Whatever.) - self.compiler.create_static_lib(objects, lib_name, - output_dir=self.build_clib, - debug=self.debug) diff --git a/Lib/distutils/command/build_ext.py b/Lib/distutils/command/build_ext.py deleted file mode 100644 index acf2fc5484a..00000000000 --- a/Lib/distutils/command/build_ext.py +++ /dev/null @@ -1,755 +0,0 @@ -"""distutils.command.build_ext - -Implements the Distutils 'build_ext' command, for building extension -modules (currently limited to C extensions, should accommodate C++ -extensions ASAP).""" - -import contextlib -import os -import re -import sys -from distutils.core import Command -from distutils.errors import * -from distutils.sysconfig import customize_compiler, get_python_version -from distutils.sysconfig import get_config_h_filename -from distutils.dep_util import newer_group -from distutils.extension import Extension -from distutils.util import get_platform -from distutils import log - -from site import USER_BASE - -# An extension name is just a dot-separated list of Python NAMEs (ie. -# the same as a fully-qualified module name). -extension_name_re = re.compile \ - (r'^[a-zA-Z_][a-zA-Z_0-9]*(\.[a-zA-Z_][a-zA-Z_0-9]*)*$') - - -def show_compilers (): - from distutils.ccompiler import show_compilers - show_compilers() - - -class build_ext(Command): - - description = "build C/C++ extensions (compile/link to build directory)" - - # XXX thoughts on how to deal with complex command-line options like - # these, i.e. how to make it so fancy_getopt can suck them off the - # command line and make it look like setup.py defined the appropriate - # lists of tuples of what-have-you. - # - each command needs a callback to process its command-line options - # - Command.__init__() needs access to its share of the whole - # command line (must ultimately come from - # Distribution.parse_command_line()) - # - it then calls the current command class' option-parsing - # callback to deal with weird options like -D, which have to - # parse the option text and churn out some custom data - # structure - # - that data structure (in this case, a list of 2-tuples) - # will then be present in the command object by the time - # we get to finalize_options() (i.e. the constructor - # takes care of both command-line and client options - # in between initialize_options() and finalize_options()) - - sep_by = " (separated by '%s')" % os.pathsep - user_options = [ - ('build-lib=', 'b', - "directory for compiled extension modules"), - ('build-temp=', 't', - "directory for temporary files (build by-products)"), - ('plat-name=', 'p', - "platform name to cross-compile for, if supported " - "(default: %s)" % get_platform()), - ('inplace', 'i', - "ignore build-lib and put compiled extensions into the source " + - "directory alongside your pure Python modules"), - ('include-dirs=', 'I', - "list of directories to search for header files" + sep_by), - ('define=', 'D', - "C preprocessor macros to define"), - ('undef=', 'U', - "C preprocessor macros to undefine"), - ('libraries=', 'l', - "external C libraries to link with"), - ('library-dirs=', 'L', - "directories to search for external C libraries" + sep_by), - ('rpath=', 'R', - "directories to search for shared C libraries at runtime"), - ('link-objects=', 'O', - "extra explicit link objects to include in the link"), - ('debug', 'g', - "compile/link with debugging information"), - ('force', 'f', - "forcibly build everything (ignore file timestamps)"), - ('compiler=', 'c', - "specify the compiler type"), - ('parallel=', 'j', - "number of parallel build jobs"), - ('swig-cpp', None, - "make SWIG create C++ files (default is C)"), - ('swig-opts=', None, - "list of SWIG command line options"), - ('swig=', None, - "path to the SWIG executable"), - ('user', None, - "add user include, library and rpath") - ] - - boolean_options = ['inplace', 'debug', 'force', 'swig-cpp', 'user'] - - help_options = [ - ('help-compiler', None, - "list available compilers", show_compilers), - ] - - def initialize_options(self): - self.extensions = None - self.build_lib = None - self.plat_name = None - self.build_temp = None - self.inplace = 0 - self.package = None - - self.include_dirs = None - self.define = None - self.undef = None - self.libraries = None - self.library_dirs = None - self.rpath = None - self.link_objects = None - self.debug = None - self.force = None - self.compiler = None - self.swig = None - self.swig_cpp = None - self.swig_opts = None - self.user = None - self.parallel = None - - def finalize_options(self): - from distutils import sysconfig - - self.set_undefined_options('build', - ('build_lib', 'build_lib'), - ('build_temp', 'build_temp'), - ('compiler', 'compiler'), - ('debug', 'debug'), - ('force', 'force'), - ('parallel', 'parallel'), - ('plat_name', 'plat_name'), - ) - - if self.package is None: - self.package = self.distribution.ext_package - - self.extensions = self.distribution.ext_modules - - # Make sure Python's include directories (for Python.h, pyconfig.h, - # etc.) are in the include search path. - py_include = sysconfig.get_python_inc() - plat_py_include = sysconfig.get_python_inc(plat_specific=1) - if self.include_dirs is None: - self.include_dirs = self.distribution.include_dirs or [] - if isinstance(self.include_dirs, str): - self.include_dirs = self.include_dirs.split(os.pathsep) - - # If in a virtualenv, add its include directory - # Issue 16116 - if sys.exec_prefix != sys.base_exec_prefix: - self.include_dirs.append(os.path.join(sys.exec_prefix, 'include')) - - # Put the Python "system" include dir at the end, so that - # any local include dirs take precedence. - self.include_dirs.append(py_include) - if plat_py_include != py_include: - self.include_dirs.append(plat_py_include) - - self.ensure_string_list('libraries') - self.ensure_string_list('link_objects') - - # Life is easier if we're not forever checking for None, so - # simplify these options to empty lists if unset - if self.libraries is None: - self.libraries = [] - if self.library_dirs is None: - self.library_dirs = [] - elif isinstance(self.library_dirs, str): - self.library_dirs = self.library_dirs.split(os.pathsep) - - if self.rpath is None: - self.rpath = [] - elif isinstance(self.rpath, str): - self.rpath = self.rpath.split(os.pathsep) - - # for extensions under windows use different directories - # for Release and Debug builds. - # also Python's library directory must be appended to library_dirs - if os.name == 'nt': - # the 'libs' directory is for binary installs - we assume that - # must be the *native* platform. But we don't really support - # cross-compiling via a binary install anyway, so we let it go. - self.library_dirs.append(os.path.join(sys.exec_prefix, 'libs')) - if sys.base_exec_prefix != sys.prefix: # Issue 16116 - self.library_dirs.append(os.path.join(sys.base_exec_prefix, 'libs')) - if self.debug: - self.build_temp = os.path.join(self.build_temp, "Debug") - else: - self.build_temp = os.path.join(self.build_temp, "Release") - - # Append the source distribution include and library directories, - # this allows distutils on windows to work in the source tree - self.include_dirs.append(os.path.dirname(get_config_h_filename())) - _sys_home = getattr(sys, '_home', None) - if _sys_home: - self.library_dirs.append(_sys_home) - - # Use the .lib files for the correct architecture - if self.plat_name == 'win32': - suffix = 'win32' - else: - # win-amd64 or win-ia64 - suffix = self.plat_name[4:] - new_lib = os.path.join(sys.exec_prefix, 'PCbuild') - if suffix: - new_lib = os.path.join(new_lib, suffix) - self.library_dirs.append(new_lib) - - # for extensions under Cygwin and AtheOS Python's library directory must be - # appended to library_dirs - if sys.platform[:6] == 'cygwin' or sys.platform[:6] == 'atheos': - if sys.executable.startswith(os.path.join(sys.exec_prefix, "bin")): - # building third party extensions - self.library_dirs.append(os.path.join(sys.prefix, "lib", - "python" + get_python_version(), - "config")) - else: - # building python standard extensions - self.library_dirs.append('.') - - # For building extensions with a shared Python library, - # Python's library directory must be appended to library_dirs - # See Issues: #1600860, #4366 - if False and (sysconfig.get_config_var('Py_ENABLE_SHARED')): - if not sysconfig.python_build: - # building third party extensions - self.library_dirs.append(sysconfig.get_config_var('LIBDIR')) - else: - # building python standard extensions - self.library_dirs.append('.') - - # The argument parsing will result in self.define being a string, but - # it has to be a list of 2-tuples. All the preprocessor symbols - # specified by the 'define' option will be set to '1'. Multiple - # symbols can be separated with commas. - - if self.define: - defines = self.define.split(',') - self.define = [(symbol, '1') for symbol in defines] - - # The option for macros to undefine is also a string from the - # option parsing, but has to be a list. Multiple symbols can also - # be separated with commas here. - if self.undef: - self.undef = self.undef.split(',') - - if self.swig_opts is None: - self.swig_opts = [] - else: - self.swig_opts = self.swig_opts.split(' ') - - # Finally add the user include and library directories if requested - if self.user: - user_include = os.path.join(USER_BASE, "include") - user_lib = os.path.join(USER_BASE, "lib") - if os.path.isdir(user_include): - self.include_dirs.append(user_include) - if os.path.isdir(user_lib): - self.library_dirs.append(user_lib) - self.rpath.append(user_lib) - - if isinstance(self.parallel, str): - try: - self.parallel = int(self.parallel) - except ValueError: - raise DistutilsOptionError("parallel should be an integer") - - def run(self): - from distutils.ccompiler import new_compiler - - # 'self.extensions', as supplied by setup.py, is a list of - # Extension instances. See the documentation for Extension (in - # distutils.extension) for details. - # - # For backwards compatibility with Distutils 0.8.2 and earlier, we - # also allow the 'extensions' list to be a list of tuples: - # (ext_name, build_info) - # where build_info is a dictionary containing everything that - # Extension instances do except the name, with a few things being - # differently named. We convert these 2-tuples to Extension - # instances as needed. - - if not self.extensions: - return - - # If we were asked to build any C/C++ libraries, make sure that the - # directory where we put them is in the library search path for - # linking extensions. - if self.distribution.has_c_libraries(): - build_clib = self.get_finalized_command('build_clib') - self.libraries.extend(build_clib.get_library_names() or []) - self.library_dirs.append(build_clib.build_clib) - - # Setup the CCompiler object that we'll use to do all the - # compiling and linking - self.compiler = new_compiler(compiler=self.compiler, - verbose=self.verbose, - dry_run=self.dry_run, - force=self.force) - customize_compiler(self.compiler) - # If we are cross-compiling, init the compiler now (if we are not - # cross-compiling, init would not hurt, but people may rely on - # late initialization of compiler even if they shouldn't...) - if os.name == 'nt' and self.plat_name != get_platform(): - self.compiler.initialize(self.plat_name) - - # And make sure that any compile/link-related options (which might - # come from the command-line or from the setup script) are set in - # that CCompiler object -- that way, they automatically apply to - # all compiling and linking done here. - if self.include_dirs is not None: - self.compiler.set_include_dirs(self.include_dirs) - if self.define is not None: - # 'define' option is a list of (name,value) tuples - for (name, value) in self.define: - self.compiler.define_macro(name, value) - if self.undef is not None: - for macro in self.undef: - self.compiler.undefine_macro(macro) - if self.libraries is not None: - self.compiler.set_libraries(self.libraries) - if self.library_dirs is not None: - self.compiler.set_library_dirs(self.library_dirs) - if self.rpath is not None: - self.compiler.set_runtime_library_dirs(self.rpath) - if self.link_objects is not None: - self.compiler.set_link_objects(self.link_objects) - - # Now actually compile and link everything. - self.build_extensions() - - def check_extensions_list(self, extensions): - """Ensure that the list of extensions (presumably provided as a - command option 'extensions') is valid, i.e. it is a list of - Extension objects. We also support the old-style list of 2-tuples, - where the tuples are (ext_name, build_info), which are converted to - Extension instances here. - - Raise DistutilsSetupError if the structure is invalid anywhere; - just returns otherwise. - """ - if not isinstance(extensions, list): - raise DistutilsSetupError( - "'ext_modules' option must be a list of Extension instances") - - for i, ext in enumerate(extensions): - if isinstance(ext, Extension): - continue # OK! (assume type-checking done - # by Extension constructor) - - if not isinstance(ext, tuple) or len(ext) != 2: - raise DistutilsSetupError( - "each element of 'ext_modules' option must be an " - "Extension instance or 2-tuple") - - ext_name, build_info = ext - - log.warn("old-style (ext_name, build_info) tuple found in " - "ext_modules for extension '%s' " - "-- please convert to Extension instance", ext_name) - - if not (isinstance(ext_name, str) and - extension_name_re.match(ext_name)): - raise DistutilsSetupError( - "first element of each tuple in 'ext_modules' " - "must be the extension name (a string)") - - if not isinstance(build_info, dict): - raise DistutilsSetupError( - "second element of each tuple in 'ext_modules' " - "must be a dictionary (build info)") - - # OK, the (ext_name, build_info) dict is type-safe: convert it - # to an Extension instance. - ext = Extension(ext_name, build_info['sources']) - - # Easy stuff: one-to-one mapping from dict elements to - # instance attributes. - for key in ('include_dirs', 'library_dirs', 'libraries', - 'extra_objects', 'extra_compile_args', - 'extra_link_args'): - val = build_info.get(key) - if val is not None: - setattr(ext, key, val) - - # Medium-easy stuff: same syntax/semantics, different names. - ext.runtime_library_dirs = build_info.get('rpath') - if 'def_file' in build_info: - log.warn("'def_file' element of build info dict " - "no longer supported") - - # Non-trivial stuff: 'macros' split into 'define_macros' - # and 'undef_macros'. - macros = build_info.get('macros') - if macros: - ext.define_macros = [] - ext.undef_macros = [] - for macro in macros: - if not (isinstance(macro, tuple) and len(macro) in (1, 2)): - raise DistutilsSetupError( - "'macros' element of build info dict " - "must be 1- or 2-tuple") - if len(macro) == 1: - ext.undef_macros.append(macro[0]) - elif len(macro) == 2: - ext.define_macros.append(macro) - - extensions[i] = ext - - def get_source_files(self): - self.check_extensions_list(self.extensions) - filenames = [] - - # Wouldn't it be neat if we knew the names of header files too... - for ext in self.extensions: - filenames.extend(ext.sources) - return filenames - - def get_outputs(self): - # Sanity check the 'extensions' list -- can't assume this is being - # done in the same run as a 'build_extensions()' call (in fact, we - # can probably assume that it *isn't*!). - self.check_extensions_list(self.extensions) - - # And build the list of output (built) filenames. Note that this - # ignores the 'inplace' flag, and assumes everything goes in the - # "build" tree. - outputs = [] - for ext in self.extensions: - outputs.append(self.get_ext_fullpath(ext.name)) - return outputs - - def build_extensions(self): - # First, sanity-check the 'extensions' list - self.check_extensions_list(self.extensions) - if self.parallel: - self._build_extensions_parallel() - else: - self._build_extensions_serial() - - def _build_extensions_parallel(self): - workers = self.parallel - if self.parallel is True: - workers = os.cpu_count() # may return None - try: - from concurrent.futures import ThreadPoolExecutor - except ImportError: - workers = None - - if workers is None: - self._build_extensions_serial() - return - - with ThreadPoolExecutor(max_workers=workers) as executor: - futures = [executor.submit(self.build_extension, ext) - for ext in self.extensions] - for ext, fut in zip(self.extensions, futures): - with self._filter_build_errors(ext): - fut.result() - - def _build_extensions_serial(self): - for ext in self.extensions: - with self._filter_build_errors(ext): - self.build_extension(ext) - - @contextlib.contextmanager - def _filter_build_errors(self, ext): - try: - yield - except (CCompilerError, DistutilsError, CompileError) as e: - if not ext.optional: - raise - self.warn('building extension "%s" failed: %s' % - (ext.name, e)) - - def build_extension(self, ext): - sources = ext.sources - if sources is None or not isinstance(sources, (list, tuple)): - raise DistutilsSetupError( - "in 'ext_modules' option (extension '%s'), " - "'sources' must be present and must be " - "a list of source filenames" % ext.name) - sources = list(sources) - - ext_path = self.get_ext_fullpath(ext.name) - depends = sources + ext.depends - if not (self.force or newer_group(depends, ext_path, 'newer')): - log.debug("skipping '%s' extension (up-to-date)", ext.name) - return - else: - log.info("building '%s' extension", ext.name) - - # First, scan the sources for SWIG definition files (.i), run - # SWIG on 'em to create .c files, and modify the sources list - # accordingly. - sources = self.swig_sources(sources, ext) - - # Next, compile the source code to object files. - - # XXX not honouring 'define_macros' or 'undef_macros' -- the - # CCompiler API needs to change to accommodate this, and I - # want to do one thing at a time! - - # Two possible sources for extra compiler arguments: - # - 'extra_compile_args' in Extension object - # - CFLAGS environment variable (not particularly - # elegant, but people seem to expect it and I - # guess it's useful) - # The environment variable should take precedence, and - # any sensible compiler will give precedence to later - # command line args. Hence we combine them in order: - extra_args = ext.extra_compile_args or [] - - macros = ext.define_macros[:] - for undef in ext.undef_macros: - macros.append((undef,)) - - objects = self.compiler.compile(sources, - output_dir=self.build_temp, - macros=macros, - include_dirs=ext.include_dirs, - debug=self.debug, - extra_postargs=extra_args, - depends=ext.depends) - - # XXX outdated variable, kept here in case third-part code - # needs it. - self._built_objects = objects[:] - - # Now link the object files together into a "shared object" -- - # of course, first we have to figure out all the other things - # that go into the mix. - if ext.extra_objects: - objects.extend(ext.extra_objects) - extra_args = ext.extra_link_args or [] - - # Detect target language, if not provided - language = ext.language or self.compiler.detect_language(sources) - - self.compiler.link_shared_object( - objects, ext_path, - libraries=self.get_libraries(ext), - library_dirs=ext.library_dirs, - runtime_library_dirs=ext.runtime_library_dirs, - extra_postargs=extra_args, - export_symbols=self.get_export_symbols(ext), - debug=self.debug, - build_temp=self.build_temp, - target_lang=language) - - def swig_sources(self, sources, extension): - """Walk the list of source files in 'sources', looking for SWIG - interface (.i) files. Run SWIG on all that are found, and - return a modified 'sources' list with SWIG source files replaced - by the generated C (or C++) files. - """ - new_sources = [] - swig_sources = [] - swig_targets = {} - - # XXX this drops generated C/C++ files into the source tree, which - # is fine for developers who want to distribute the generated - # source -- but there should be an option to put SWIG output in - # the temp dir. - - if self.swig_cpp: - log.warn("--swig-cpp is deprecated - use --swig-opts=-c++") - - if self.swig_cpp or ('-c++' in self.swig_opts) or \ - ('-c++' in extension.swig_opts): - target_ext = '.cpp' - else: - target_ext = '.c' - - for source in sources: - (base, ext) = os.path.splitext(source) - if ext == ".i": # SWIG interface file - new_sources.append(base + '_wrap' + target_ext) - swig_sources.append(source) - swig_targets[source] = new_sources[-1] - else: - new_sources.append(source) - - if not swig_sources: - return new_sources - - swig = self.swig or self.find_swig() - swig_cmd = [swig, "-python"] - swig_cmd.extend(self.swig_opts) - if self.swig_cpp: - swig_cmd.append("-c++") - - # Do not override commandline arguments - if not self.swig_opts: - for o in extension.swig_opts: - swig_cmd.append(o) - - for source in swig_sources: - target = swig_targets[source] - log.info("swigging %s to %s", source, target) - self.spawn(swig_cmd + ["-o", target, source]) - - return new_sources - - def find_swig(self): - """Return the name of the SWIG executable. On Unix, this is - just "swig" -- it should be in the PATH. Tries a bit harder on - Windows. - """ - if os.name == "posix": - return "swig" - elif os.name == "nt": - # Look for SWIG in its standard installation directory on - # Windows (or so I presume!). If we find it there, great; - # if not, act like Unix and assume it's in the PATH. - for vers in ("1.3", "1.2", "1.1"): - fn = os.path.join("c:\\swig%s" % vers, "swig.exe") - if os.path.isfile(fn): - return fn - else: - return "swig.exe" - else: - raise DistutilsPlatformError( - "I don't know how to find (much less run) SWIG " - "on platform '%s'" % os.name) - - # -- Name generators ----------------------------------------------- - # (extension names, filenames, whatever) - def get_ext_fullpath(self, ext_name): - """Returns the path of the filename for a given extension. - - The file is located in `build_lib` or directly in the package - (inplace option). - """ - fullname = self.get_ext_fullname(ext_name) - modpath = fullname.split('.') - filename = self.get_ext_filename(modpath[-1]) - - if not self.inplace: - # no further work needed - # returning : - # build_dir/package/path/filename - filename = os.path.join(*modpath[:-1]+[filename]) - return os.path.join(self.build_lib, filename) - - # the inplace option requires to find the package directory - # using the build_py command for that - package = '.'.join(modpath[0:-1]) - build_py = self.get_finalized_command('build_py') - package_dir = os.path.abspath(build_py.get_package_dir(package)) - - # returning - # package_dir/filename - return os.path.join(package_dir, filename) - - def get_ext_fullname(self, ext_name): - """Returns the fullname of a given extension name. - - Adds the `package.` prefix""" - if self.package is None: - return ext_name - else: - return self.package + '.' + ext_name - - def get_ext_filename(self, ext_name): - r"""Convert the name of an extension (eg. "foo.bar") into the name - of the file from which it will be loaded (eg. "foo/bar.so", or - "foo\bar.pyd"). - """ - from distutils.sysconfig import get_config_var - ext_path = ext_name.split('.') - ext_suffix = get_config_var('EXT_SUFFIX') - return os.path.join(*ext_path) + ext_suffix - - def get_export_symbols(self, ext): - """Return the list of symbols that a shared extension has to - export. This either uses 'ext.export_symbols' or, if it's not - provided, "PyInit_" + module_name. Only relevant on Windows, where - the .pyd file (DLL) must export the module "PyInit_" function. - """ - initfunc_name = "PyInit_" + ext.name.split('.')[-1] - if initfunc_name not in ext.export_symbols: - ext.export_symbols.append(initfunc_name) - return ext.export_symbols - - def get_libraries(self, ext): - """Return the list of libraries to link against when building a - shared extension. On most platforms, this is just 'ext.libraries'; - on Windows, we add the Python library (eg. python20.dll). - """ - # The python library is always needed on Windows. For MSVC, this - # is redundant, since the library is mentioned in a pragma in - # pyconfig.h that MSVC groks. The other Windows compilers all seem - # to need it mentioned explicitly, though, so that's what we do. - # Append '_d' to the python import library on debug builds. - if sys.platform == "win32": - from distutils._msvccompiler import MSVCCompiler - if not isinstance(self.compiler, MSVCCompiler): - template = "python%d%d" - if self.debug: - template = template + '_d' - pythonlib = (template % - (sys.hexversion >> 24, (sys.hexversion >> 16) & 0xff)) - # don't extend ext.libraries, it may be shared with other - # extensions, it is a reference to the original list - return ext.libraries + [pythonlib] - else: - return ext.libraries - elif sys.platform[:6] == "cygwin": - template = "python%d.%d" - pythonlib = (template % - (sys.hexversion >> 24, (sys.hexversion >> 16) & 0xff)) - # don't extend ext.libraries, it may be shared with other - # extensions, it is a reference to the original list - return ext.libraries + [pythonlib] - elif sys.platform[:6] == "atheos": - from distutils import sysconfig - - template = "python%d.%d" - pythonlib = (template % - (sys.hexversion >> 24, (sys.hexversion >> 16) & 0xff)) - # Get SHLIBS from Makefile - extra = [] - for lib in sysconfig.get_config_var('SHLIBS').split(): - if lib.startswith('-l'): - extra.append(lib[2:]) - else: - extra.append(lib) - # don't extend ext.libraries, it may be shared with other - # extensions, it is a reference to the original list - return ext.libraries + [pythonlib, "m"] + extra - elif sys.platform == 'darwin': - # Don't use the default code below - return ext.libraries - elif sys.platform[:3] == 'aix': - # Don't use the default code below - return ext.libraries - else: - from distutils import sysconfig - if False and sysconfig.get_config_var('Py_ENABLE_SHARED'): - pythonlib = 'python{}.{}{}'.format( - sys.hexversion >> 24, (sys.hexversion >> 16) & 0xff, - sysconfig.get_config_var('ABIFLAGS')) - return ext.libraries + [pythonlib] - else: - return ext.libraries diff --git a/Lib/distutils/command/build_py.py b/Lib/distutils/command/build_py.py deleted file mode 100644 index cf0ca57c320..00000000000 --- a/Lib/distutils/command/build_py.py +++ /dev/null @@ -1,416 +0,0 @@ -"""distutils.command.build_py - -Implements the Distutils 'build_py' command.""" - -import os -import importlib.util -import sys -from glob import glob - -from distutils.core import Command -from distutils.errors import * -from distutils.util import convert_path, Mixin2to3 -from distutils import log - -class build_py (Command): - - description = "\"build\" pure Python modules (copy to build directory)" - - user_options = [ - ('build-lib=', 'd', "directory to \"build\" (copy) to"), - ('compile', 'c', "compile .py to .pyc"), - ('no-compile', None, "don't compile .py files [default]"), - ('optimize=', 'O', - "also compile with optimization: -O1 for \"python -O\", " - "-O2 for \"python -OO\", and -O0 to disable [default: -O0]"), - ('force', 'f', "forcibly build everything (ignore file timestamps)"), - ] - - boolean_options = ['compile', 'force'] - negative_opt = {'no-compile' : 'compile'} - - def initialize_options(self): - self.build_lib = None - self.py_modules = None - self.package = None - self.package_data = None - self.package_dir = None - self.compile = 0 - self.optimize = 0 - self.force = None - - def finalize_options(self): - self.set_undefined_options('build', - ('build_lib', 'build_lib'), - ('force', 'force')) - - # Get the distribution options that are aliases for build_py - # options -- list of packages and list of modules. - self.packages = self.distribution.packages - self.py_modules = self.distribution.py_modules - self.package_data = self.distribution.package_data - self.package_dir = {} - if self.distribution.package_dir: - for name, path in self.distribution.package_dir.items(): - self.package_dir[name] = convert_path(path) - self.data_files = self.get_data_files() - - # Ick, copied straight from install_lib.py (fancy_getopt needs a - # type system! Hell, *everything* needs a type system!!!) - if not isinstance(self.optimize, int): - try: - self.optimize = int(self.optimize) - assert 0 <= self.optimize <= 2 - except (ValueError, AssertionError): - raise DistutilsOptionError("optimize must be 0, 1, or 2") - - def run(self): - # XXX copy_file by default preserves atime and mtime. IMHO this is - # the right thing to do, but perhaps it should be an option -- in - # particular, a site administrator might want installed files to - # reflect the time of installation rather than the last - # modification time before the installed release. - - # XXX copy_file by default preserves mode, which appears to be the - # wrong thing to do: if a file is read-only in the working - # directory, we want it to be installed read/write so that the next - # installation of the same module distribution can overwrite it - # without problems. (This might be a Unix-specific issue.) Thus - # we turn off 'preserve_mode' when copying to the build directory, - # since the build directory is supposed to be exactly what the - # installation will look like (ie. we preserve mode when - # installing). - - # Two options control which modules will be installed: 'packages' - # and 'py_modules'. The former lets us work with whole packages, not - # specifying individual modules at all; the latter is for - # specifying modules one-at-a-time. - - if self.py_modules: - self.build_modules() - if self.packages: - self.build_packages() - self.build_package_data() - - self.byte_compile(self.get_outputs(include_bytecode=0)) - - def get_data_files(self): - """Generate list of '(package,src_dir,build_dir,filenames)' tuples""" - data = [] - if not self.packages: - return data - for package in self.packages: - # Locate package source directory - src_dir = self.get_package_dir(package) - - # Compute package build directory - build_dir = os.path.join(*([self.build_lib] + package.split('.'))) - - # Length of path to strip from found files - plen = 0 - if src_dir: - plen = len(src_dir)+1 - - # Strip directory from globbed filenames - filenames = [ - file[plen:] for file in self.find_data_files(package, src_dir) - ] - data.append((package, src_dir, build_dir, filenames)) - return data - - def find_data_files(self, package, src_dir): - """Return filenames for package's data files in 'src_dir'""" - globs = (self.package_data.get('', []) - + self.package_data.get(package, [])) - files = [] - for pattern in globs: - # Each pattern has to be converted to a platform-specific path - filelist = glob(os.path.join(src_dir, convert_path(pattern))) - # Files that match more than one pattern are only added once - files.extend([fn for fn in filelist if fn not in files - and os.path.isfile(fn)]) - return files - - def build_package_data(self): - """Copy data files into build directory""" - lastdir = None - for package, src_dir, build_dir, filenames in self.data_files: - for filename in filenames: - target = os.path.join(build_dir, filename) - self.mkpath(os.path.dirname(target)) - self.copy_file(os.path.join(src_dir, filename), target, - preserve_mode=False) - - def get_package_dir(self, package): - """Return the directory, relative to the top of the source - distribution, where package 'package' should be found - (at least according to the 'package_dir' option, if any).""" - path = package.split('.') - - if not self.package_dir: - if path: - return os.path.join(*path) - else: - return '' - else: - tail = [] - while path: - try: - pdir = self.package_dir['.'.join(path)] - except KeyError: - tail.insert(0, path[-1]) - del path[-1] - else: - tail.insert(0, pdir) - return os.path.join(*tail) - else: - # Oops, got all the way through 'path' without finding a - # match in package_dir. If package_dir defines a directory - # for the root (nameless) package, then fallback on it; - # otherwise, we might as well have not consulted - # package_dir at all, as we just use the directory implied - # by 'tail' (which should be the same as the original value - # of 'path' at this point). - pdir = self.package_dir.get('') - if pdir is not None: - tail.insert(0, pdir) - - if tail: - return os.path.join(*tail) - else: - return '' - - def check_package(self, package, package_dir): - # Empty dir name means current directory, which we can probably - # assume exists. Also, os.path.exists and isdir don't know about - # my "empty string means current dir" convention, so we have to - # circumvent them. - if package_dir != "": - if not os.path.exists(package_dir): - raise DistutilsFileError( - "package directory '%s' does not exist" % package_dir) - if not os.path.isdir(package_dir): - raise DistutilsFileError( - "supposed package directory '%s' exists, " - "but is not a directory" % package_dir) - - # Require __init__.py for all but the "root package" - if package: - init_py = os.path.join(package_dir, "__init__.py") - if os.path.isfile(init_py): - return init_py - else: - log.warn(("package init file '%s' not found " + - "(or not a regular file)"), init_py) - - # Either not in a package at all (__init__.py not expected), or - # __init__.py doesn't exist -- so don't return the filename. - return None - - def check_module(self, module, module_file): - if not os.path.isfile(module_file): - log.warn("file %s (for module %s) not found", module_file, module) - return False - else: - return True - - def find_package_modules(self, package, package_dir): - self.check_package(package, package_dir) - module_files = glob(os.path.join(package_dir, "*.py")) - modules = [] - setup_script = os.path.abspath(self.distribution.script_name) - - for f in module_files: - abs_f = os.path.abspath(f) - if abs_f != setup_script: - module = os.path.splitext(os.path.basename(f))[0] - modules.append((package, module, f)) - else: - self.debug_print("excluding %s" % setup_script) - return modules - - def find_modules(self): - """Finds individually-specified Python modules, ie. those listed by - module name in 'self.py_modules'. Returns a list of tuples (package, - module_base, filename): 'package' is a tuple of the path through - package-space to the module; 'module_base' is the bare (no - packages, no dots) module name, and 'filename' is the path to the - ".py" file (relative to the distribution root) that implements the - module. - """ - # Map package names to tuples of useful info about the package: - # (package_dir, checked) - # package_dir - the directory where we'll find source files for - # this package - # checked - true if we have checked that the package directory - # is valid (exists, contains __init__.py, ... ?) - packages = {} - - # List of (package, module, filename) tuples to return - modules = [] - - # We treat modules-in-packages almost the same as toplevel modules, - # just the "package" for a toplevel is empty (either an empty - # string or empty list, depending on context). Differences: - # - don't check for __init__.py in directory for empty package - for module in self.py_modules: - path = module.split('.') - package = '.'.join(path[0:-1]) - module_base = path[-1] - - try: - (package_dir, checked) = packages[package] - except KeyError: - package_dir = self.get_package_dir(package) - checked = 0 - - if not checked: - init_py = self.check_package(package, package_dir) - packages[package] = (package_dir, 1) - if init_py: - modules.append((package, "__init__", init_py)) - - # XXX perhaps we should also check for just .pyc files - # (so greedy closed-source bastards can distribute Python - # modules too) - module_file = os.path.join(package_dir, module_base + ".py") - if not self.check_module(module, module_file): - continue - - modules.append((package, module_base, module_file)) - - return modules - - def find_all_modules(self): - """Compute the list of all modules that will be built, whether - they are specified one-module-at-a-time ('self.py_modules') or - by whole packages ('self.packages'). Return a list of tuples - (package, module, module_file), just like 'find_modules()' and - 'find_package_modules()' do.""" - modules = [] - if self.py_modules: - modules.extend(self.find_modules()) - if self.packages: - for package in self.packages: - package_dir = self.get_package_dir(package) - m = self.find_package_modules(package, package_dir) - modules.extend(m) - return modules - - def get_source_files(self): - return [module[-1] for module in self.find_all_modules()] - - def get_module_outfile(self, build_dir, package, module): - outfile_path = [build_dir] + list(package) + [module + ".py"] - return os.path.join(*outfile_path) - - def get_outputs(self, include_bytecode=1): - modules = self.find_all_modules() - outputs = [] - for (package, module, module_file) in modules: - package = package.split('.') - filename = self.get_module_outfile(self.build_lib, package, module) - outputs.append(filename) - if include_bytecode: - if self.compile: - outputs.append(importlib.util.cache_from_source( - filename, optimization='')) - if self.optimize > 0: - outputs.append(importlib.util.cache_from_source( - filename, optimization=self.optimize)) - - outputs += [ - os.path.join(build_dir, filename) - for package, src_dir, build_dir, filenames in self.data_files - for filename in filenames - ] - - return outputs - - def build_module(self, module, module_file, package): - if isinstance(package, str): - package = package.split('.') - elif not isinstance(package, (list, tuple)): - raise TypeError( - "'package' must be a string (dot-separated), list, or tuple") - - # Now put the module source file into the "build" area -- this is - # easy, we just copy it somewhere under self.build_lib (the build - # directory for Python source). - outfile = self.get_module_outfile(self.build_lib, package, module) - dir = os.path.dirname(outfile) - self.mkpath(dir) - return self.copy_file(module_file, outfile, preserve_mode=0) - - def build_modules(self): - modules = self.find_modules() - for (package, module, module_file) in modules: - # Now "build" the module -- ie. copy the source file to - # self.build_lib (the build directory for Python source). - # (Actually, it gets copied to the directory for this package - # under self.build_lib.) - self.build_module(module, module_file, package) - - def build_packages(self): - for package in self.packages: - # Get list of (package, module, module_file) tuples based on - # scanning the package directory. 'package' is only included - # in the tuple so that 'find_modules()' and - # 'find_package_tuples()' have a consistent interface; it's - # ignored here (apart from a sanity check). Also, 'module' is - # the *unqualified* module name (ie. no dots, no package -- we - # already know its package!), and 'module_file' is the path to - # the .py file, relative to the current directory - # (ie. including 'package_dir'). - package_dir = self.get_package_dir(package) - modules = self.find_package_modules(package, package_dir) - - # Now loop over the modules we found, "building" each one (just - # copy it to self.build_lib). - for (package_, module, module_file) in modules: - assert package == package_ - self.build_module(module, module_file, package) - - def byte_compile(self, files): - if sys.dont_write_bytecode: - self.warn('byte-compiling is disabled, skipping.') - return - - from distutils.util import byte_compile - prefix = self.build_lib - if prefix[-1] != os.sep: - prefix = prefix + os.sep - - # XXX this code is essentially the same as the 'byte_compile() - # method of the "install_lib" command, except for the determination - # of the 'prefix' string. Hmmm. - if self.compile: - byte_compile(files, optimize=0, - force=self.force, prefix=prefix, dry_run=self.dry_run) - if self.optimize > 0: - byte_compile(files, optimize=self.optimize, - force=self.force, prefix=prefix, dry_run=self.dry_run) - -class build_py_2to3(build_py, Mixin2to3): - def run(self): - self.updated_files = [] - - # Base class code - if self.py_modules: - self.build_modules() - if self.packages: - self.build_packages() - self.build_package_data() - - # 2to3 - self.run_2to3(self.updated_files) - - # Remaining base class code - self.byte_compile(self.get_outputs(include_bytecode=0)) - - def build_module(self, module, module_file, package): - res = build_py.build_module(self, module, module_file, package) - if res[1]: - # file was copied - self.updated_files.append(res[0]) - return res diff --git a/Lib/distutils/command/build_scripts.py b/Lib/distutils/command/build_scripts.py deleted file mode 100644 index ccc70e64650..00000000000 --- a/Lib/distutils/command/build_scripts.py +++ /dev/null @@ -1,160 +0,0 @@ -"""distutils.command.build_scripts - -Implements the Distutils 'build_scripts' command.""" - -import os, re -from stat import ST_MODE -from distutils import sysconfig -from distutils.core import Command -from distutils.dep_util import newer -from distutils.util import convert_path, Mixin2to3 -from distutils import log -import tokenize - -# check if Python is called on the first line with this expression -first_line_re = re.compile(b'^#!.*python[0-9.]*([ \t].*)?$') - -class build_scripts(Command): - - description = "\"build\" scripts (copy and fixup #! line)" - - user_options = [ - ('build-dir=', 'd', "directory to \"build\" (copy) to"), - ('force', 'f', "forcibly build everything (ignore file timestamps"), - ('executable=', 'e', "specify final destination interpreter path"), - ] - - boolean_options = ['force'] - - - def initialize_options(self): - self.build_dir = None - self.scripts = None - self.force = None - self.executable = None - self.outfiles = None - - def finalize_options(self): - self.set_undefined_options('build', - ('build_scripts', 'build_dir'), - ('force', 'force'), - ('executable', 'executable')) - self.scripts = self.distribution.scripts - - def get_source_files(self): - return self.scripts - - def run(self): - if not self.scripts: - return - self.copy_scripts() - - - def copy_scripts(self): - r"""Copy each script listed in 'self.scripts'; if it's marked as a - Python script in the Unix way (first line matches 'first_line_re', - ie. starts with "\#!" and contains "python"), then adjust the first - line to refer to the current Python interpreter as we copy. - """ - self.mkpath(self.build_dir) - outfiles = [] - updated_files = [] - for script in self.scripts: - adjust = False - script = convert_path(script) - outfile = os.path.join(self.build_dir, os.path.basename(script)) - outfiles.append(outfile) - - if not self.force and not newer(script, outfile): - log.debug("not copying %s (up-to-date)", script) - continue - - # Always open the file, but ignore failures in dry-run mode -- - # that way, we'll get accurate feedback if we can read the - # script. - try: - f = open(script, "rb") - except OSError: - if not self.dry_run: - raise - f = None - else: - encoding, lines = tokenize.detect_encoding(f.readline) - f.seek(0) - first_line = f.readline() - if not first_line: - self.warn("%s is an empty file (skipping)" % script) - continue - - match = first_line_re.match(first_line) - if match: - adjust = True - post_interp = match.group(1) or b'' - - if adjust: - log.info("copying and adjusting %s -> %s", script, - self.build_dir) - updated_files.append(outfile) - if not self.dry_run: - if not sysconfig.python_build: - executable = self.executable - else: - executable = os.path.join( - sysconfig.get_config_var("BINDIR"), - "python%s%s" % (sysconfig.get_config_var("VERSION"), - sysconfig.get_config_var("EXE"))) - executable = os.fsencode(executable) - shebang = b"#!" + executable + post_interp + b"\n" - # Python parser starts to read a script using UTF-8 until - # it gets a #coding:xxx cookie. The shebang has to be the - # first line of a file, the #coding:xxx cookie cannot be - # written before. So the shebang has to be decodable from - # UTF-8. - try: - shebang.decode('utf-8') - except UnicodeDecodeError: - raise ValueError( - "The shebang ({!r}) is not decodable " - "from utf-8".format(shebang)) - # If the script is encoded to a custom encoding (use a - # #coding:xxx cookie), the shebang has to be decodable from - # the script encoding too. - try: - shebang.decode(encoding) - except UnicodeDecodeError: - raise ValueError( - "The shebang ({!r}) is not decodable " - "from the script encoding ({})" - .format(shebang, encoding)) - with open(outfile, "wb") as outf: - outf.write(shebang) - outf.writelines(f.readlines()) - if f: - f.close() - else: - if f: - f.close() - updated_files.append(outfile) - self.copy_file(script, outfile) - - if os.name == 'posix': - for file in outfiles: - if self.dry_run: - log.info("changing mode of %s", file) - else: - oldmode = os.stat(file)[ST_MODE] & 0o7777 - newmode = (oldmode | 0o555) & 0o7777 - if newmode != oldmode: - log.info("changing mode of %s from %o to %o", - file, oldmode, newmode) - os.chmod(file, newmode) - # XXX should we modify self.outfiles? - return outfiles, updated_files - -class build_scripts_2to3(build_scripts, Mixin2to3): - - def copy_scripts(self): - outfiles, updated_files = build_scripts.copy_scripts(self) - if not self.dry_run: - self.run_2to3(updated_files) - return outfiles, updated_files diff --git a/Lib/distutils/command/check.py b/Lib/distutils/command/check.py deleted file mode 100644 index 7ebe707cff4..00000000000 --- a/Lib/distutils/command/check.py +++ /dev/null @@ -1,145 +0,0 @@ -"""distutils.command.check - -Implements the Distutils 'check' command. -""" -from distutils.core import Command -from distutils.errors import DistutilsSetupError - -try: - # docutils is installed - from docutils.utils import Reporter - from docutils.parsers.rst import Parser - from docutils import frontend - from docutils import nodes - from io import StringIO - - class SilentReporter(Reporter): - - def __init__(self, source, report_level, halt_level, stream=None, - debug=0, encoding='ascii', error_handler='replace'): - self.messages = [] - Reporter.__init__(self, source, report_level, halt_level, stream, - debug, encoding, error_handler) - - def system_message(self, level, message, *children, **kwargs): - self.messages.append((level, message, children, kwargs)) - return nodes.system_message(message, level=level, - type=self.levels[level], - *children, **kwargs) - - HAS_DOCUTILS = True -except Exception: - # Catch all exceptions because exceptions besides ImportError probably - # indicate that docutils is not ported to Py3k. - HAS_DOCUTILS = False - -class check(Command): - """This command checks the meta-data of the package. - """ - description = ("perform some checks on the package") - user_options = [('metadata', 'm', 'Verify meta-data'), - ('restructuredtext', 'r', - ('Checks if long string meta-data syntax ' - 'are reStructuredText-compliant')), - ('strict', 's', - 'Will exit with an error if a check fails')] - - boolean_options = ['metadata', 'restructuredtext', 'strict'] - - def initialize_options(self): - """Sets default values for options.""" - self.restructuredtext = 0 - self.metadata = 1 - self.strict = 0 - self._warnings = 0 - - def finalize_options(self): - pass - - def warn(self, msg): - """Counts the number of warnings that occurs.""" - self._warnings += 1 - return Command.warn(self, msg) - - def run(self): - """Runs the command.""" - # perform the various tests - if self.metadata: - self.check_metadata() - if self.restructuredtext: - if HAS_DOCUTILS: - self.check_restructuredtext() - elif self.strict: - raise DistutilsSetupError('The docutils package is needed.') - - # let's raise an error in strict mode, if we have at least - # one warning - if self.strict and self._warnings > 0: - raise DistutilsSetupError('Please correct your package.') - - def check_metadata(self): - """Ensures that all required elements of meta-data are supplied. - - name, version, URL, (author and author_email) or - (maintainer and maintainer_email)). - - Warns if any are missing. - """ - metadata = self.distribution.metadata - - missing = [] - for attr in ('name', 'version', 'url'): - if not (hasattr(metadata, attr) and getattr(metadata, attr)): - missing.append(attr) - - if missing: - self.warn("missing required meta-data: %s" % ', '.join(missing)) - if metadata.author: - if not metadata.author_email: - self.warn("missing meta-data: if 'author' supplied, " + - "'author_email' must be supplied too") - elif metadata.maintainer: - if not metadata.maintainer_email: - self.warn("missing meta-data: if 'maintainer' supplied, " + - "'maintainer_email' must be supplied too") - else: - self.warn("missing meta-data: either (author and author_email) " + - "or (maintainer and maintainer_email) " + - "must be supplied") - - def check_restructuredtext(self): - """Checks if the long string fields are reST-compliant.""" - data = self.distribution.get_long_description() - for warning in self._check_rst_data(data): - line = warning[-1].get('line') - if line is None: - warning = warning[1] - else: - warning = '%s (line %s)' % (warning[1], line) - self.warn(warning) - - def _check_rst_data(self, data): - """Returns warnings when the provided data doesn't compile.""" - source_path = StringIO() - parser = Parser() - settings = frontend.OptionParser(components=(Parser,)).get_default_values() - settings.tab_width = 4 - settings.pep_references = None - settings.rfc_references = None - reporter = SilentReporter(source_path, - settings.report_level, - settings.halt_level, - stream=settings.warning_stream, - debug=settings.debug, - encoding=settings.error_encoding, - error_handler=settings.error_encoding_error_handler) - - document = nodes.document(settings, reporter, source=source_path) - document.note_source(source_path, -1) - try: - parser.parse(data, document) - except AttributeError as e: - reporter.messages.append( - (-1, 'Could not finish the parsing: %s.' % e, '', {})) - - return reporter.messages diff --git a/Lib/distutils/command/clean.py b/Lib/distutils/command/clean.py deleted file mode 100644 index 0cb27016621..00000000000 --- a/Lib/distutils/command/clean.py +++ /dev/null @@ -1,76 +0,0 @@ -"""distutils.command.clean - -Implements the Distutils 'clean' command.""" - -# contributed by Bastian Kleineidam , added 2000-03-18 - -import os -from distutils.core import Command -from distutils.dir_util import remove_tree -from distutils import log - -class clean(Command): - - description = "clean up temporary files from 'build' command" - user_options = [ - ('build-base=', 'b', - "base build directory (default: 'build.build-base')"), - ('build-lib=', None, - "build directory for all modules (default: 'build.build-lib')"), - ('build-temp=', 't', - "temporary build directory (default: 'build.build-temp')"), - ('build-scripts=', None, - "build directory for scripts (default: 'build.build-scripts')"), - ('bdist-base=', None, - "temporary directory for built distributions"), - ('all', 'a', - "remove all build output, not just temporary by-products") - ] - - boolean_options = ['all'] - - def initialize_options(self): - self.build_base = None - self.build_lib = None - self.build_temp = None - self.build_scripts = None - self.bdist_base = None - self.all = None - - def finalize_options(self): - self.set_undefined_options('build', - ('build_base', 'build_base'), - ('build_lib', 'build_lib'), - ('build_scripts', 'build_scripts'), - ('build_temp', 'build_temp')) - self.set_undefined_options('bdist', - ('bdist_base', 'bdist_base')) - - def run(self): - # remove the build/temp. directory (unless it's already - # gone) - if os.path.exists(self.build_temp): - remove_tree(self.build_temp, dry_run=self.dry_run) - else: - log.debug("'%s' does not exist -- can't clean it", - self.build_temp) - - if self.all: - # remove build directories - for directory in (self.build_lib, - self.bdist_base, - self.build_scripts): - if os.path.exists(directory): - remove_tree(directory, dry_run=self.dry_run) - else: - log.warn("'%s' does not exist -- can't clean it", - directory) - - # just for the heck of it, try to remove the base build directory: - # we might have emptied it right now, but if not we don't care - if not self.dry_run: - try: - os.rmdir(self.build_base) - log.info("removing '%s'", self.build_base) - except OSError: - pass diff --git a/Lib/distutils/command/command_template b/Lib/distutils/command/command_template deleted file mode 100644 index 6106819db84..00000000000 --- a/Lib/distutils/command/command_template +++ /dev/null @@ -1,33 +0,0 @@ -"""distutils.command.x - -Implements the Distutils 'x' command. -""" - -# created 2000/mm/dd, John Doe - -__revision__ = "$Id$" - -from distutils.core import Command - - -class x(Command): - - # Brief (40-50 characters) description of the command - description = "" - - # List of option tuples: long name, short name (None if no short - # name), and help string. - user_options = [('', '', - ""), - ] - - def initialize_options(self): - self. = None - self. = None - self. = None - - def finalize_options(self): - if self.x is None: - self.x = - - def run(self): diff --git a/Lib/distutils/command/config.py b/Lib/distutils/command/config.py deleted file mode 100644 index 4ae153d1943..00000000000 --- a/Lib/distutils/command/config.py +++ /dev/null @@ -1,347 +0,0 @@ -"""distutils.command.config - -Implements the Distutils 'config' command, a (mostly) empty command class -that exists mainly to be sub-classed by specific module distributions and -applications. The idea is that while every "config" command is different, -at least they're all named the same, and users always see "config" in the -list of standard commands. Also, this is a good place to put common -configure-like tasks: "try to compile this C code", or "figure out where -this header file lives". -""" - -import os, re - -from distutils.core import Command -from distutils.errors import DistutilsExecError -from distutils.sysconfig import customize_compiler -from distutils import log - -LANG_EXT = {"c": ".c", "c++": ".cxx"} - -class config(Command): - - description = "prepare to build" - - user_options = [ - ('compiler=', None, - "specify the compiler type"), - ('cc=', None, - "specify the compiler executable"), - ('include-dirs=', 'I', - "list of directories to search for header files"), - ('define=', 'D', - "C preprocessor macros to define"), - ('undef=', 'U', - "C preprocessor macros to undefine"), - ('libraries=', 'l', - "external C libraries to link with"), - ('library-dirs=', 'L', - "directories to search for external C libraries"), - - ('noisy', None, - "show every action (compile, link, run, ...) taken"), - ('dump-source', None, - "dump generated source files before attempting to compile them"), - ] - - - # The three standard command methods: since the "config" command - # does nothing by default, these are empty. - - def initialize_options(self): - self.compiler = None - self.cc = None - self.include_dirs = None - self.libraries = None - self.library_dirs = None - - # maximal output for now - self.noisy = 1 - self.dump_source = 1 - - # list of temporary files generated along-the-way that we have - # to clean at some point - self.temp_files = [] - - def finalize_options(self): - if self.include_dirs is None: - self.include_dirs = self.distribution.include_dirs or [] - elif isinstance(self.include_dirs, str): - self.include_dirs = self.include_dirs.split(os.pathsep) - - if self.libraries is None: - self.libraries = [] - elif isinstance(self.libraries, str): - self.libraries = [self.libraries] - - if self.library_dirs is None: - self.library_dirs = [] - elif isinstance(self.library_dirs, str): - self.library_dirs = self.library_dirs.split(os.pathsep) - - def run(self): - pass - - # Utility methods for actual "config" commands. The interfaces are - # loosely based on Autoconf macros of similar names. Sub-classes - # may use these freely. - - def _check_compiler(self): - """Check that 'self.compiler' really is a CCompiler object; - if not, make it one. - """ - # We do this late, and only on-demand, because this is an expensive - # import. - from distutils.ccompiler import CCompiler, new_compiler - if not isinstance(self.compiler, CCompiler): - self.compiler = new_compiler(compiler=self.compiler, - dry_run=self.dry_run, force=1) - customize_compiler(self.compiler) - if self.include_dirs: - self.compiler.set_include_dirs(self.include_dirs) - if self.libraries: - self.compiler.set_libraries(self.libraries) - if self.library_dirs: - self.compiler.set_library_dirs(self.library_dirs) - - def _gen_temp_sourcefile(self, body, headers, lang): - filename = "_configtest" + LANG_EXT[lang] - file = open(filename, "w") - if headers: - for header in headers: - file.write("#include <%s>\n" % header) - file.write("\n") - file.write(body) - if body[-1] != "\n": - file.write("\n") - file.close() - return filename - - def _preprocess(self, body, headers, include_dirs, lang): - src = self._gen_temp_sourcefile(body, headers, lang) - out = "_configtest.i" - self.temp_files.extend([src, out]) - self.compiler.preprocess(src, out, include_dirs=include_dirs) - return (src, out) - - def _compile(self, body, headers, include_dirs, lang): - src = self._gen_temp_sourcefile(body, headers, lang) - if self.dump_source: - dump_file(src, "compiling '%s':" % src) - (obj,) = self.compiler.object_filenames([src]) - self.temp_files.extend([src, obj]) - self.compiler.compile([src], include_dirs=include_dirs) - return (src, obj) - - def _link(self, body, headers, include_dirs, libraries, library_dirs, - lang): - (src, obj) = self._compile(body, headers, include_dirs, lang) - prog = os.path.splitext(os.path.basename(src))[0] - self.compiler.link_executable([obj], prog, - libraries=libraries, - library_dirs=library_dirs, - target_lang=lang) - - if self.compiler.exe_extension is not None: - prog = prog + self.compiler.exe_extension - self.temp_files.append(prog) - - return (src, obj, prog) - - def _clean(self, *filenames): - if not filenames: - filenames = self.temp_files - self.temp_files = [] - log.info("removing: %s", ' '.join(filenames)) - for filename in filenames: - try: - os.remove(filename) - except OSError: - pass - - - # XXX these ignore the dry-run flag: what to do, what to do? even if - # you want a dry-run build, you still need some sort of configuration - # info. My inclination is to make it up to the real config command to - # consult 'dry_run', and assume a default (minimal) configuration if - # true. The problem with trying to do it here is that you'd have to - # return either true or false from all the 'try' methods, neither of - # which is correct. - - # XXX need access to the header search path and maybe default macros. - - def try_cpp(self, body=None, headers=None, include_dirs=None, lang="c"): - """Construct a source file from 'body' (a string containing lines - of C/C++ code) and 'headers' (a list of header files to include) - and run it through the preprocessor. Return true if the - preprocessor succeeded, false if there were any errors. - ('body' probably isn't of much use, but what the heck.) - """ - from distutils.ccompiler import CompileError - self._check_compiler() - ok = True - try: - self._preprocess(body, headers, include_dirs, lang) - except CompileError: - ok = False - - self._clean() - return ok - - def search_cpp(self, pattern, body=None, headers=None, include_dirs=None, - lang="c"): - """Construct a source file (just like 'try_cpp()'), run it through - the preprocessor, and return true if any line of the output matches - 'pattern'. 'pattern' should either be a compiled regex object or a - string containing a regex. If both 'body' and 'headers' are None, - preprocesses an empty file -- which can be useful to determine the - symbols the preprocessor and compiler set by default. - """ - self._check_compiler() - src, out = self._preprocess(body, headers, include_dirs, lang) - - if isinstance(pattern, str): - pattern = re.compile(pattern) - - file = open(out) - match = False - while True: - line = file.readline() - if line == '': - break - if pattern.search(line): - match = True - break - - file.close() - self._clean() - return match - - def try_compile(self, body, headers=None, include_dirs=None, lang="c"): - """Try to compile a source file built from 'body' and 'headers'. - Return true on success, false otherwise. - """ - from distutils.ccompiler import CompileError - self._check_compiler() - try: - self._compile(body, headers, include_dirs, lang) - ok = True - except CompileError: - ok = False - - log.info(ok and "success!" or "failure.") - self._clean() - return ok - - def try_link(self, body, headers=None, include_dirs=None, libraries=None, - library_dirs=None, lang="c"): - """Try to compile and link a source file, built from 'body' and - 'headers', to executable form. Return true on success, false - otherwise. - """ - from distutils.ccompiler import CompileError, LinkError - self._check_compiler() - try: - self._link(body, headers, include_dirs, - libraries, library_dirs, lang) - ok = True - except (CompileError, LinkError): - ok = False - - log.info(ok and "success!" or "failure.") - self._clean() - return ok - - def try_run(self, body, headers=None, include_dirs=None, libraries=None, - library_dirs=None, lang="c"): - """Try to compile, link to an executable, and run a program - built from 'body' and 'headers'. Return true on success, false - otherwise. - """ - from distutils.ccompiler import CompileError, LinkError - self._check_compiler() - try: - src, obj, exe = self._link(body, headers, include_dirs, - libraries, library_dirs, lang) - self.spawn([exe]) - ok = True - except (CompileError, LinkError, DistutilsExecError): - ok = False - - log.info(ok and "success!" or "failure.") - self._clean() - return ok - - - # -- High-level methods -------------------------------------------- - # (these are the ones that are actually likely to be useful - # when implementing a real-world config command!) - - def check_func(self, func, headers=None, include_dirs=None, - libraries=None, library_dirs=None, decl=0, call=0): - """Determine if function 'func' is available by constructing a - source file that refers to 'func', and compiles and links it. - If everything succeeds, returns true; otherwise returns false. - - The constructed source file starts out by including the header - files listed in 'headers'. If 'decl' is true, it then declares - 'func' (as "int func()"); you probably shouldn't supply 'headers' - and set 'decl' true in the same call, or you might get errors about - a conflicting declarations for 'func'. Finally, the constructed - 'main()' function either references 'func' or (if 'call' is true) - calls it. 'libraries' and 'library_dirs' are used when - linking. - """ - self._check_compiler() - body = [] - if decl: - body.append("int %s ();" % func) - body.append("int main () {") - if call: - body.append(" %s();" % func) - else: - body.append(" %s;" % func) - body.append("}") - body = "\n".join(body) + "\n" - - return self.try_link(body, headers, include_dirs, - libraries, library_dirs) - - def check_lib(self, library, library_dirs=None, headers=None, - include_dirs=None, other_libraries=[]): - """Determine if 'library' is available to be linked against, - without actually checking that any particular symbols are provided - by it. 'headers' will be used in constructing the source file to - be compiled, but the only effect of this is to check if all the - header files listed are available. Any libraries listed in - 'other_libraries' will be included in the link, in case 'library' - has symbols that depend on other libraries. - """ - self._check_compiler() - return self.try_link("int main (void) { }", headers, include_dirs, - [library] + other_libraries, library_dirs) - - def check_header(self, header, include_dirs=None, library_dirs=None, - lang="c"): - """Determine if the system header file named by 'header_file' - exists and can be found by the preprocessor; return true if so, - false otherwise. - """ - return self.try_cpp(body="/* No body */", headers=[header], - include_dirs=include_dirs) - - -def dump_file(filename, head=None): - """Dumps a file content into log.info. - - If head is not None, will be dumped before the file content. - """ - if head is None: - log.info('%s', filename) - else: - log.info(head) - file = open(filename) - try: - log.info(file.read()) - finally: - file.close() diff --git a/Lib/distutils/command/install.py b/Lib/distutils/command/install.py deleted file mode 100644 index fd3357ea78e..00000000000 --- a/Lib/distutils/command/install.py +++ /dev/null @@ -1,705 +0,0 @@ -"""distutils.command.install - -Implements the Distutils 'install' command.""" - -import sys -import os - -from distutils import log -from distutils.core import Command -from distutils.debug import DEBUG -from distutils.sysconfig import get_config_vars -from distutils.errors import DistutilsPlatformError -from distutils.file_util import write_file -from distutils.util import convert_path, subst_vars, change_root -from distutils.util import get_platform -from distutils.errors import DistutilsOptionError - -from site import USER_BASE -from site import USER_SITE -HAS_USER_SITE = True - -WINDOWS_SCHEME = { - 'purelib': '$base/Lib/site-packages', - 'platlib': '$base/Lib/site-packages', - 'headers': '$base/Include/$dist_name', - 'scripts': '$base/Scripts', - 'data' : '$base', -} - -INSTALL_SCHEMES = { - 'unix_prefix': { - 'purelib': '$base/lib/python$py_version_short/site-packages', - 'platlib': '$platbase/lib/python$py_version_short/site-packages', - 'headers': '$base/include/python$py_version_short$abiflags/$dist_name', - 'scripts': '$base/bin', - 'data' : '$base', - }, - 'unix_local': { - 'purelib': '$base/local/lib/python$py_version_short/dist-packages', - 'platlib': '$platbase/local/lib/python$py_version_short/dist-packages', - 'headers': '$base/local/include/python$py_version_short/$dist_name', - 'scripts': '$base/local/bin', - 'data' : '$base/local', - }, - 'deb_system': { - 'purelib': '$base/lib/python3/dist-packages', - 'platlib': '$platbase/lib/python3/dist-packages', - 'headers': '$base/include/python$py_version_short/$dist_name', - 'scripts': '$base/bin', - 'data' : '$base', - }, - 'unix_home': { - 'purelib': '$base/lib/python', - 'platlib': '$base/lib/python', - 'headers': '$base/include/python/$dist_name', - 'scripts': '$base/bin', - 'data' : '$base', - }, - 'nt': WINDOWS_SCHEME, - } - -# user site schemes -if HAS_USER_SITE: - INSTALL_SCHEMES['nt_user'] = { - 'purelib': '$usersite', - 'platlib': '$usersite', - 'headers': '$userbase/Python$py_version_nodot/Include/$dist_name', - 'scripts': '$userbase/Python$py_version_nodot/Scripts', - 'data' : '$userbase', - } - - INSTALL_SCHEMES['unix_user'] = { - 'purelib': '$usersite', - 'platlib': '$usersite', - 'headers': - '$userbase/include/python$py_version_short$abiflags/$dist_name', - 'scripts': '$userbase/bin', - 'data' : '$userbase', - } - -# XXX RUSTPYTHON: replace python with rustpython in all these paths -for group in INSTALL_SCHEMES.values(): - for key in group.keys(): - group[key] = group[key].replace("Python", "RustPython").replace("python", "rustpython") - -# The keys to an installation scheme; if any new types of files are to be -# installed, be sure to add an entry to every installation scheme above, -# and to SCHEME_KEYS here. -SCHEME_KEYS = ('purelib', 'platlib', 'headers', 'scripts', 'data') - - -class install(Command): - - description = "install everything from build directory" - - user_options = [ - # Select installation scheme and set base director(y|ies) - ('prefix=', None, - "installation prefix"), - ('exec-prefix=', None, - "(Unix only) prefix for platform-specific files"), - ('home=', None, - "(Unix only) home directory to install under"), - - # Or, just set the base director(y|ies) - ('install-base=', None, - "base installation directory (instead of --prefix or --home)"), - ('install-platbase=', None, - "base installation directory for platform-specific files " + - "(instead of --exec-prefix or --home)"), - ('root=', None, - "install everything relative to this alternate root directory"), - - # Or, explicitly set the installation scheme - ('install-purelib=', None, - "installation directory for pure Python module distributions"), - ('install-platlib=', None, - "installation directory for non-pure module distributions"), - ('install-lib=', None, - "installation directory for all module distributions " + - "(overrides --install-purelib and --install-platlib)"), - - ('install-headers=', None, - "installation directory for C/C++ headers"), - ('install-scripts=', None, - "installation directory for Python scripts"), - ('install-data=', None, - "installation directory for data files"), - - # Byte-compilation options -- see install_lib.py for details, as - # these are duplicated from there (but only install_lib does - # anything with them). - ('compile', 'c', "compile .py to .pyc [default]"), - ('no-compile', None, "don't compile .py files"), - ('optimize=', 'O', - "also compile with optimization: -O1 for \"python -O\", " - "-O2 for \"python -OO\", and -O0 to disable [default: -O0]"), - - # Miscellaneous control options - ('force', 'f', - "force installation (overwrite any existing files)"), - ('skip-build', None, - "skip rebuilding everything (for testing/debugging)"), - - # Where to install documentation (eventually!) - #('doc-format=', None, "format of documentation to generate"), - #('install-man=', None, "directory for Unix man pages"), - #('install-html=', None, "directory for HTML documentation"), - #('install-info=', None, "directory for GNU info files"), - - ('record=', None, - "filename in which to record list of installed files"), - - ('install-layout=', None, - "installation layout to choose (known values: deb, unix)"), - ] - - boolean_options = ['compile', 'force', 'skip-build'] - - if HAS_USER_SITE: - user_options.append(('user', None, - "install in user site-package '%s'" % USER_SITE)) - boolean_options.append('user') - - negative_opt = {'no-compile' : 'compile'} - - - def initialize_options(self): - """Initializes options.""" - # High-level options: these select both an installation base - # and scheme. - self.prefix = None - self.exec_prefix = None - self.home = None - self.user = 0 - self.prefix_option = None - - # These select only the installation base; it's up to the user to - # specify the installation scheme (currently, that means supplying - # the --install-{platlib,purelib,scripts,data} options). - self.install_base = None - self.install_platbase = None - self.root = None - - # These options are the actual installation directories; if not - # supplied by the user, they are filled in using the installation - # scheme implied by prefix/exec-prefix/home and the contents of - # that installation scheme. - self.install_purelib = None # for pure module distributions - self.install_platlib = None # non-pure (dists w/ extensions) - self.install_headers = None # for C/C++ headers - self.install_lib = None # set to either purelib or platlib - self.install_scripts = None - self.install_data = None - self.install_userbase = USER_BASE - self.install_usersite = USER_SITE - - # enable custom installation, known values: deb - self.install_layout = None - self.multiarch = None - - self.compile = None - self.optimize = None - - # Deprecated - # These two are for putting non-packagized distributions into their - # own directory and creating a .pth file if it makes sense. - # 'extra_path' comes from the setup file; 'install_path_file' can - # be turned off if it makes no sense to install a .pth file. (But - # better to install it uselessly than to guess wrong and not - # install it when it's necessary and would be used!) Currently, - # 'install_path_file' is always true unless some outsider meddles - # with it. - self.extra_path = None - self.install_path_file = 1 - - # 'force' forces installation, even if target files are not - # out-of-date. 'skip_build' skips running the "build" command, - # handy if you know it's not necessary. 'warn_dir' (which is *not* - # a user option, it's just there so the bdist_* commands can turn - # it off) determines whether we warn about installing to a - # directory not in sys.path. - self.force = 0 - self.skip_build = 0 - self.warn_dir = 1 - - # These are only here as a conduit from the 'build' command to the - # 'install_*' commands that do the real work. ('build_base' isn't - # actually used anywhere, but it might be useful in future.) They - # are not user options, because if the user told the install - # command where the build directory is, that wouldn't affect the - # build command. - self.build_base = None - self.build_lib = None - - # Not defined yet because we don't know anything about - # documentation yet. - #self.install_man = None - #self.install_html = None - #self.install_info = None - - self.record = None - - - # -- Option finalizing methods ------------------------------------- - # (This is rather more involved than for most commands, - # because this is where the policy for installing third- - # party Python modules on various platforms given a wide - # array of user input is decided. Yes, it's quite complex!) - - def finalize_options(self): - """Finalizes options.""" - # This method (and its pliant slaves, like 'finalize_unix()', - # 'finalize_other()', and 'select_scheme()') is where the default - # installation directories for modules, extension modules, and - # anything else we care to install from a Python module - # distribution. Thus, this code makes a pretty important policy - # statement about how third-party stuff is added to a Python - # installation! Note that the actual work of installation is done - # by the relatively simple 'install_*' commands; they just take - # their orders from the installation directory options determined - # here. - - # Check for errors/inconsistencies in the options; first, stuff - # that's wrong on any platform. - - if ((self.prefix or self.exec_prefix or self.home) and - (self.install_base or self.install_platbase)): - raise DistutilsOptionError( - "must supply either prefix/exec-prefix/home or " + - "install-base/install-platbase -- not both") - - if self.home and (self.prefix or self.exec_prefix): - raise DistutilsOptionError( - "must supply either home or prefix/exec-prefix -- not both") - - if self.user and (self.prefix or self.exec_prefix or self.home or - self.install_base or self.install_platbase): - raise DistutilsOptionError("can't combine user with prefix, " - "exec_prefix/home, or install_(plat)base") - - # Next, stuff that's wrong (or dubious) only on certain platforms. - if os.name != "posix": - if self.exec_prefix: - self.warn("exec-prefix option ignored on this platform") - self.exec_prefix = None - - # Now the interesting logic -- so interesting that we farm it out - # to other methods. The goal of these methods is to set the final - # values for the install_{lib,scripts,data,...} options, using as - # input a heady brew of prefix, exec_prefix, home, install_base, - # install_platbase, user-supplied versions of - # install_{purelib,platlib,lib,scripts,data,...}, and the - # INSTALL_SCHEME dictionary above. Phew! - - self.dump_dirs("pre-finalize_{unix,other}") - - if os.name == 'posix': - self.finalize_unix() - else: - self.finalize_other() - - self.dump_dirs("post-finalize_{unix,other}()") - - # Expand configuration variables, tilde, etc. in self.install_base - # and self.install_platbase -- that way, we can use $base or - # $platbase in the other installation directories and not worry - # about needing recursive variable expansion (shudder). - - py_version = sys.version.split()[0] - (prefix, exec_prefix) = get_config_vars('prefix', 'exec_prefix') - try: - abiflags = sys.abiflags - except AttributeError: - # sys.abiflags may not be defined on all platforms. - abiflags = '' - self.config_vars = {'dist_name': self.distribution.get_name(), - 'dist_version': self.distribution.get_version(), - 'dist_fullname': self.distribution.get_fullname(), - 'py_version': py_version, - 'py_version_short': '%d.%d' % sys.version_info[:2], - 'py_version_nodot': '%d%d' % sys.version_info[:2], - 'sys_prefix': prefix, - 'prefix': prefix, - 'sys_exec_prefix': exec_prefix, - 'exec_prefix': exec_prefix, - 'abiflags': abiflags, - } - - if HAS_USER_SITE: - self.config_vars['userbase'] = self.install_userbase - self.config_vars['usersite'] = self.install_usersite - - self.expand_basedirs() - - self.dump_dirs("post-expand_basedirs()") - - # Now define config vars for the base directories so we can expand - # everything else. - self.config_vars['base'] = self.install_base - self.config_vars['platbase'] = self.install_platbase - - if DEBUG: - from pprint import pprint - print("config vars:") - pprint(self.config_vars) - - # Expand "~" and configuration variables in the installation - # directories. - self.expand_dirs() - - self.dump_dirs("post-expand_dirs()") - - # Create directories in the home dir: - if self.user: - self.create_home_path() - - # Pick the actual directory to install all modules to: either - # install_purelib or install_platlib, depending on whether this - # module distribution is pure or not. Of course, if the user - # already specified install_lib, use their selection. - if self.install_lib is None: - if self.distribution.ext_modules: # has extensions: non-pure - self.install_lib = self.install_platlib - else: - self.install_lib = self.install_purelib - - - # Convert directories from Unix /-separated syntax to the local - # convention. - self.convert_paths('lib', 'purelib', 'platlib', - 'scripts', 'data', 'headers', - 'userbase', 'usersite') - - # Deprecated - # Well, we're not actually fully completely finalized yet: we still - # have to deal with 'extra_path', which is the hack for allowing - # non-packagized module distributions (hello, Numerical Python!) to - # get their own directories. - self.handle_extra_path() - self.install_libbase = self.install_lib # needed for .pth file - self.install_lib = os.path.join(self.install_lib, self.extra_dirs) - - # If a new root directory was supplied, make all the installation - # dirs relative to it. - if self.root is not None: - self.change_roots('libbase', 'lib', 'purelib', 'platlib', - 'scripts', 'data', 'headers') - - self.dump_dirs("after prepending root") - - # Find out the build directories, ie. where to install from. - self.set_undefined_options('build', - ('build_base', 'build_base'), - ('build_lib', 'build_lib')) - - # Punt on doc directories for now -- after all, we're punting on - # documentation completely! - - def dump_dirs(self, msg): - """Dumps the list of user options.""" - if not DEBUG: - return - from distutils.fancy_getopt import longopt_xlate - log.debug(msg + ":") - for opt in self.user_options: - opt_name = opt[0] - if opt_name[-1] == "=": - opt_name = opt_name[0:-1] - if opt_name in self.negative_opt: - opt_name = self.negative_opt[opt_name] - opt_name = opt_name.translate(longopt_xlate) - val = not getattr(self, opt_name) - else: - opt_name = opt_name.translate(longopt_xlate) - val = getattr(self, opt_name) - log.debug(" %s: %s", opt_name, val) - - def finalize_unix(self): - """Finalizes options for posix platforms.""" - if self.install_base is not None or self.install_platbase is not None: - if ((self.install_lib is None and - self.install_purelib is None and - self.install_platlib is None) or - self.install_headers is None or - self.install_scripts is None or - self.install_data is None): - raise DistutilsOptionError( - "install-base or install-platbase supplied, but " - "installation scheme is incomplete") - return - - if self.user: - if self.install_userbase is None: - raise DistutilsPlatformError( - "User base directory is not specified") - self.install_base = self.install_platbase = self.install_userbase - self.select_scheme("unix_user") - elif self.home is not None: - self.install_base = self.install_platbase = self.home - self.select_scheme("unix_home") - else: - self.prefix_option = self.prefix - if self.prefix is None: - if self.exec_prefix is not None: - raise DistutilsOptionError( - "must not supply exec-prefix without prefix") - - self.prefix = os.path.normpath(sys.prefix) - self.exec_prefix = os.path.normpath(sys.exec_prefix) - - else: - if self.exec_prefix is None: - self.exec_prefix = self.prefix - - self.install_base = self.prefix - self.install_platbase = self.exec_prefix - if self.install_layout: - if self.install_layout.lower() in ['deb']: - import sysconfig - self.multiarch = sysconfig.get_config_var('MULTIARCH') - self.select_scheme("deb_system") - elif self.install_layout.lower() in ['unix']: - self.select_scheme("unix_prefix") - else: - raise DistutilsOptionError( - "unknown value for --install-layout") - elif ((self.prefix_option and - os.path.normpath(self.prefix) != '/usr/local') - or sys.base_prefix != sys.prefix - or 'PYTHONUSERBASE' in os.environ - or 'VIRTUAL_ENV' in os.environ - or 'real_prefix' in sys.__dict__): - self.select_scheme("unix_prefix") - else: - if os.path.normpath(self.prefix) == '/usr/local': - self.prefix = self.exec_prefix = '/usr' - self.install_base = self.install_platbase = '/usr' - self.select_scheme("unix_local") - - def finalize_other(self): - """Finalizes options for non-posix platforms""" - if self.user: - if self.install_userbase is None: - raise DistutilsPlatformError( - "User base directory is not specified") - self.install_base = self.install_platbase = self.install_userbase - self.select_scheme(os.name + "_user") - elif self.home is not None: - self.install_base = self.install_platbase = self.home - self.select_scheme("unix_home") - else: - if self.prefix is None: - self.prefix = os.path.normpath(sys.prefix) - - self.install_base = self.install_platbase = self.prefix - try: - self.select_scheme(os.name) - except KeyError: - raise DistutilsPlatformError( - "I don't know how to install stuff on '%s'" % os.name) - - def select_scheme(self, name): - """Sets the install directories by applying the install schemes.""" - # it's the caller's problem if they supply a bad name! - scheme = INSTALL_SCHEMES[name] - for key in SCHEME_KEYS: - attrname = 'install_' + key - if getattr(self, attrname) is None: - setattr(self, attrname, scheme[key]) - - def _expand_attrs(self, attrs): - for attr in attrs: - val = getattr(self, attr) - if val is not None: - if os.name == 'posix' or os.name == 'nt': - val = os.path.expanduser(val) - val = subst_vars(val, self.config_vars) - setattr(self, attr, val) - - def expand_basedirs(self): - """Calls `os.path.expanduser` on install_base, install_platbase and - root.""" - self._expand_attrs(['install_base', 'install_platbase', 'root']) - - def expand_dirs(self): - """Calls `os.path.expanduser` on install dirs.""" - self._expand_attrs(['install_purelib', 'install_platlib', - 'install_lib', 'install_headers', - 'install_scripts', 'install_data',]) - - def convert_paths(self, *names): - """Call `convert_path` over `names`.""" - for name in names: - attr = "install_" + name - setattr(self, attr, convert_path(getattr(self, attr))) - - def handle_extra_path(self): - """Set `path_file` and `extra_dirs` using `extra_path`.""" - if self.extra_path is None: - self.extra_path = self.distribution.extra_path - - if self.extra_path is not None: - log.warn( - "Distribution option extra_path is deprecated. " - "See issue27919 for details." - ) - if isinstance(self.extra_path, str): - self.extra_path = self.extra_path.split(',') - - if len(self.extra_path) == 1: - path_file = extra_dirs = self.extra_path[0] - elif len(self.extra_path) == 2: - path_file, extra_dirs = self.extra_path - else: - raise DistutilsOptionError( - "'extra_path' option must be a list, tuple, or " - "comma-separated string with 1 or 2 elements") - - # convert to local form in case Unix notation used (as it - # should be in setup scripts) - extra_dirs = convert_path(extra_dirs) - else: - path_file = None - extra_dirs = '' - - # XXX should we warn if path_file and not extra_dirs? (in which - # case the path file would be harmless but pointless) - self.path_file = path_file - self.extra_dirs = extra_dirs - - def change_roots(self, *names): - """Change the install directories pointed by name using root.""" - for name in names: - attr = "install_" + name - setattr(self, attr, change_root(self.root, getattr(self, attr))) - - def create_home_path(self): - """Create directories under ~.""" - if not self.user: - return - home = convert_path(os.path.expanduser("~")) - for name, path in self.config_vars.items(): - if path.startswith(home) and not os.path.isdir(path): - self.debug_print("os.makedirs('%s', 0o700)" % path) - os.makedirs(path, 0o700) - - # -- Command execution methods ------------------------------------- - - def run(self): - """Runs the command.""" - # Obviously have to build before we can install - if not self.skip_build: - self.run_command('build') - # If we built for any other platform, we can't install. - build_plat = self.distribution.get_command_obj('build').plat_name - # check warn_dir - it is a clue that the 'install' is happening - # internally, and not to sys.path, so we don't check the platform - # matches what we are running. - if self.warn_dir and build_plat != get_platform(): - raise DistutilsPlatformError("Can't install when " - "cross-compiling") - - # Run all sub-commands (at least those that need to be run) - for cmd_name in self.get_sub_commands(): - self.run_command(cmd_name) - - if self.path_file: - self.create_path_file() - - # write list of installed files, if requested. - if self.record: - outputs = self.get_outputs() - if self.root: # strip any package prefix - root_len = len(self.root) - for counter in range(len(outputs)): - outputs[counter] = outputs[counter][root_len:] - self.execute(write_file, - (self.record, outputs), - "writing list of installed files to '%s'" % - self.record) - - sys_path = map(os.path.normpath, sys.path) - sys_path = map(os.path.normcase, sys_path) - install_lib = os.path.normcase(os.path.normpath(self.install_lib)) - if (self.warn_dir and - not (self.path_file and self.install_path_file) and - install_lib not in sys_path): - log.debug(("modules installed to '%s', which is not in " - "Python's module search path (sys.path) -- " - "you'll have to change the search path yourself"), - self.install_lib) - - def create_path_file(self): - """Creates the .pth file""" - filename = os.path.join(self.install_libbase, - self.path_file + ".pth") - if self.install_path_file: - self.execute(write_file, - (filename, [self.extra_dirs]), - "creating %s" % filename) - else: - self.warn("path file '%s' not created" % filename) - - - # -- Reporting methods --------------------------------------------- - - def get_outputs(self): - """Assembles the outputs of all the sub-commands.""" - outputs = [] - for cmd_name in self.get_sub_commands(): - cmd = self.get_finalized_command(cmd_name) - # Add the contents of cmd.get_outputs(), ensuring - # that outputs doesn't contain duplicate entries - for filename in cmd.get_outputs(): - if filename not in outputs: - outputs.append(filename) - - if self.path_file and self.install_path_file: - outputs.append(os.path.join(self.install_libbase, - self.path_file + ".pth")) - - return outputs - - def get_inputs(self): - """Returns the inputs of all the sub-commands""" - # XXX gee, this looks familiar ;-( - inputs = [] - for cmd_name in self.get_sub_commands(): - cmd = self.get_finalized_command(cmd_name) - inputs.extend(cmd.get_inputs()) - - return inputs - - # -- Predicates for sub-command list ------------------------------- - - def has_lib(self): - """Returns true if the current distribution has any Python - modules to install.""" - return (self.distribution.has_pure_modules() or - self.distribution.has_ext_modules()) - - def has_headers(self): - """Returns true if the current distribution has any headers to - install.""" - return self.distribution.has_headers() - - def has_scripts(self): - """Returns true if the current distribution has any scripts to. - install.""" - return self.distribution.has_scripts() - - def has_data(self): - """Returns true if the current distribution has any data to. - install.""" - return self.distribution.has_data_files() - - # 'sub_commands': a list of commands this command might have to run to - # get its work done. See cmd.py for more info. - sub_commands = [('install_lib', has_lib), - ('install_headers', has_headers), - ('install_scripts', has_scripts), - ('install_data', has_data), - ('install_egg_info', lambda self:True), - ] diff --git a/Lib/distutils/command/install_data.py b/Lib/distutils/command/install_data.py deleted file mode 100644 index 947cd76a99e..00000000000 --- a/Lib/distutils/command/install_data.py +++ /dev/null @@ -1,79 +0,0 @@ -"""distutils.command.install_data - -Implements the Distutils 'install_data' command, for installing -platform-independent data files.""" - -# contributed by Bastian Kleineidam - -import os -from distutils.core import Command -from distutils.util import change_root, convert_path - -class install_data(Command): - - description = "install data files" - - user_options = [ - ('install-dir=', 'd', - "base directory for installing data files " - "(default: installation base dir)"), - ('root=', None, - "install everything relative to this alternate root directory"), - ('force', 'f', "force installation (overwrite existing files)"), - ] - - boolean_options = ['force'] - - def initialize_options(self): - self.install_dir = None - self.outfiles = [] - self.root = None - self.force = 0 - self.data_files = self.distribution.data_files - self.warn_dir = 1 - - def finalize_options(self): - self.set_undefined_options('install', - ('install_data', 'install_dir'), - ('root', 'root'), - ('force', 'force'), - ) - - def run(self): - self.mkpath(self.install_dir) - for f in self.data_files: - if isinstance(f, str): - # it's a simple file, so copy it - f = convert_path(f) - if self.warn_dir: - self.warn("setup script did not provide a directory for " - "'%s' -- installing right in '%s'" % - (f, self.install_dir)) - (out, _) = self.copy_file(f, self.install_dir) - self.outfiles.append(out) - else: - # it's a tuple with path to install to and a list of files - dir = convert_path(f[0]) - if not os.path.isabs(dir): - dir = os.path.join(self.install_dir, dir) - elif self.root: - dir = change_root(self.root, dir) - self.mkpath(dir) - - if f[1] == []: - # If there are no files listed, the user must be - # trying to create an empty directory, so add the - # directory to the list of output files. - self.outfiles.append(dir) - else: - # Copy files, adding them to the list of output files. - for data in f[1]: - data = convert_path(data) - (out, _) = self.copy_file(data, dir) - self.outfiles.append(out) - - def get_inputs(self): - return self.data_files or [] - - def get_outputs(self): - return self.outfiles diff --git a/Lib/distutils/command/install_egg_info.py b/Lib/distutils/command/install_egg_info.py deleted file mode 100644 index 0a71b610005..00000000000 --- a/Lib/distutils/command/install_egg_info.py +++ /dev/null @@ -1,97 +0,0 @@ -"""distutils.command.install_egg_info - -Implements the Distutils 'install_egg_info' command, for installing -a package's PKG-INFO metadata.""" - - -from distutils.cmd import Command -from distutils import log, dir_util -import os, sys, re - -class install_egg_info(Command): - """Install an .egg-info file for the package""" - - description = "Install package's PKG-INFO metadata as an .egg-info file" - user_options = [ - ('install-dir=', 'd', "directory to install to"), - ('install-layout', None, "custom installation layout"), - ] - - def initialize_options(self): - self.install_dir = None - self.install_layout = None - self.prefix_option = None - - def finalize_options(self): - self.set_undefined_options('install_lib',('install_dir','install_dir')) - self.set_undefined_options('install',('install_layout','install_layout')) - self.set_undefined_options('install',('prefix_option','prefix_option')) - if self.install_layout: - if not self.install_layout.lower() in ['deb', 'unix']: - raise DistutilsOptionError( - "unknown value for --install-layout") - no_pyver = (self.install_layout.lower() == 'deb') - elif self.prefix_option: - no_pyver = False - else: - no_pyver = True - if no_pyver: - basename = "%s-%s.egg-info" % ( - to_filename(safe_name(self.distribution.get_name())), - to_filename(safe_version(self.distribution.get_version())) - ) - else: - basename = "%s-%s-py%d.%d.egg-info" % ( - to_filename(safe_name(self.distribution.get_name())), - to_filename(safe_version(self.distribution.get_version())), - *sys.version_info[:2] - ) - self.target = os.path.join(self.install_dir, basename) - self.outputs = [self.target] - - def run(self): - target = self.target - if os.path.isdir(target) and not os.path.islink(target): - dir_util.remove_tree(target, dry_run=self.dry_run) - elif os.path.exists(target): - self.execute(os.unlink,(self.target,),"Removing "+target) - elif not os.path.isdir(self.install_dir): - self.execute(os.makedirs, (self.install_dir,), - "Creating "+self.install_dir) - log.info("Writing %s", target) - if not self.dry_run: - with open(target, 'w', encoding='UTF-8') as f: - self.distribution.metadata.write_pkg_file(f) - - def get_outputs(self): - return self.outputs - - -# The following routines are taken from setuptools' pkg_resources module and -# can be replaced by importing them from pkg_resources once it is included -# in the stdlib. - -def safe_name(name): - """Convert an arbitrary string to a standard distribution name - - Any runs of non-alphanumeric/. characters are replaced with a single '-'. - """ - return re.sub('[^A-Za-z0-9.]+', '-', name) - - -def safe_version(version): - """Convert an arbitrary string to a standard version string - - Spaces become dots, and all other non-alphanumeric characters become - dashes, with runs of multiple dashes condensed to a single dash. - """ - version = version.replace(' ','.') - return re.sub('[^A-Za-z0-9.]+', '-', version) - - -def to_filename(name): - """Convert a project or version name to its filename-escaped form - - Any '-' characters are currently replaced with '_'. - """ - return name.replace('-','_') diff --git a/Lib/distutils/command/install_headers.py b/Lib/distutils/command/install_headers.py deleted file mode 100644 index 9bb0b18dc0d..00000000000 --- a/Lib/distutils/command/install_headers.py +++ /dev/null @@ -1,47 +0,0 @@ -"""distutils.command.install_headers - -Implements the Distutils 'install_headers' command, to install C/C++ header -files to the Python include directory.""" - -from distutils.core import Command - - -# XXX force is never used -class install_headers(Command): - - description = "install C/C++ header files" - - user_options = [('install-dir=', 'd', - "directory to install header files to"), - ('force', 'f', - "force installation (overwrite existing files)"), - ] - - boolean_options = ['force'] - - def initialize_options(self): - self.install_dir = None - self.force = 0 - self.outfiles = [] - - def finalize_options(self): - self.set_undefined_options('install', - ('install_headers', 'install_dir'), - ('force', 'force')) - - - def run(self): - headers = self.distribution.headers - if not headers: - return - - self.mkpath(self.install_dir) - for header in headers: - (out, _) = self.copy_file(header, self.install_dir) - self.outfiles.append(out) - - def get_inputs(self): - return self.distribution.headers or [] - - def get_outputs(self): - return self.outfiles diff --git a/Lib/distutils/command/install_lib.py b/Lib/distutils/command/install_lib.py deleted file mode 100644 index eef63626ff7..00000000000 --- a/Lib/distutils/command/install_lib.py +++ /dev/null @@ -1,221 +0,0 @@ -"""distutils.command.install_lib - -Implements the Distutils 'install_lib' command -(install all Python modules).""" - -import os -import importlib.util -import sys - -from distutils.core import Command -from distutils.errors import DistutilsOptionError - - -# Extension for Python source files. -PYTHON_SOURCE_EXTENSION = ".py" - -class install_lib(Command): - - description = "install all Python modules (extensions and pure Python)" - - # The byte-compilation options are a tad confusing. Here are the - # possible scenarios: - # 1) no compilation at all (--no-compile --no-optimize) - # 2) compile .pyc only (--compile --no-optimize; default) - # 3) compile .pyc and "opt-1" .pyc (--compile --optimize) - # 4) compile "opt-1" .pyc only (--no-compile --optimize) - # 5) compile .pyc and "opt-2" .pyc (--compile --optimize-more) - # 6) compile "opt-2" .pyc only (--no-compile --optimize-more) - # - # The UI for this is two options, 'compile' and 'optimize'. - # 'compile' is strictly boolean, and only decides whether to - # generate .pyc files. 'optimize' is three-way (0, 1, or 2), and - # decides both whether to generate .pyc files and what level of - # optimization to use. - - user_options = [ - ('install-dir=', 'd', "directory to install to"), - ('build-dir=','b', "build directory (where to install from)"), - ('force', 'f', "force installation (overwrite existing files)"), - ('compile', 'c', "compile .py to .pyc [default]"), - ('no-compile', None, "don't compile .py files"), - ('optimize=', 'O', - "also compile with optimization: -O1 for \"python -O\", " - "-O2 for \"python -OO\", and -O0 to disable [default: -O0]"), - ('skip-build', None, "skip the build steps"), - ] - - boolean_options = ['force', 'compile', 'skip-build'] - negative_opt = {'no-compile' : 'compile'} - - def initialize_options(self): - # let the 'install' command dictate our installation directory - self.install_dir = None - self.build_dir = None - self.force = 0 - self.compile = None - self.optimize = None - self.skip_build = None - self.multiarch = None # if we should rename the extensions - - def finalize_options(self): - # Get all the information we need to install pure Python modules - # from the umbrella 'install' command -- build (source) directory, - # install (target) directory, and whether to compile .py files. - self.set_undefined_options('install', - ('build_lib', 'build_dir'), - ('install_lib', 'install_dir'), - ('force', 'force'), - ('compile', 'compile'), - ('optimize', 'optimize'), - ('skip_build', 'skip_build'), - ('multiarch', 'multiarch'), - ) - - if self.compile is None: - self.compile = True - if self.optimize is None: - self.optimize = False - - if not isinstance(self.optimize, int): - try: - self.optimize = int(self.optimize) - if self.optimize not in (0, 1, 2): - raise AssertionError - except (ValueError, AssertionError): - raise DistutilsOptionError("optimize must be 0, 1, or 2") - - def run(self): - # Make sure we have built everything we need first - self.build() - - # Install everything: simply dump the entire contents of the build - # directory to the installation directory (that's the beauty of - # having a build directory!) - outfiles = self.install() - - # (Optionally) compile .py to .pyc - if outfiles is not None and self.distribution.has_pure_modules(): - self.byte_compile(outfiles) - - # -- Top-level worker functions ------------------------------------ - # (called from 'run()') - - def build(self): - if not self.skip_build: - if self.distribution.has_pure_modules(): - self.run_command('build_py') - if self.distribution.has_ext_modules(): - self.run_command('build_ext') - - def install(self): - if os.path.isdir(self.build_dir): - import distutils.dir_util - distutils.dir_util._multiarch = self.multiarch - outfiles = self.copy_tree(self.build_dir, self.install_dir) - else: - self.warn("'%s' does not exist -- no Python modules to install" % - self.build_dir) - return - return outfiles - - def byte_compile(self, files): - if sys.dont_write_bytecode: - self.warn('byte-compiling is disabled, skipping.') - return - - from distutils.util import byte_compile - - # Get the "--root" directory supplied to the "install" command, - # and use it as a prefix to strip off the purported filename - # encoded in bytecode files. This is far from complete, but it - # should at least generate usable bytecode in RPM distributions. - install_root = self.get_finalized_command('install').root - - if self.compile: - byte_compile(files, optimize=0, - force=self.force, prefix=install_root, - dry_run=self.dry_run) - if self.optimize > 0: - byte_compile(files, optimize=self.optimize, - force=self.force, prefix=install_root, - verbose=self.verbose, dry_run=self.dry_run) - - - # -- Utility methods ----------------------------------------------- - - def _mutate_outputs(self, has_any, build_cmd, cmd_option, output_dir): - if not has_any: - return [] - - build_cmd = self.get_finalized_command(build_cmd) - build_files = build_cmd.get_outputs() - build_dir = getattr(build_cmd, cmd_option) - - prefix_len = len(build_dir) + len(os.sep) - outputs = [] - for file in build_files: - outputs.append(os.path.join(output_dir, file[prefix_len:])) - - return outputs - - def _bytecode_filenames(self, py_filenames): - bytecode_files = [] - for py_file in py_filenames: - # Since build_py handles package data installation, the - # list of outputs can contain more than just .py files. - # Make sure we only report bytecode for the .py files. - ext = os.path.splitext(os.path.normcase(py_file))[1] - if ext != PYTHON_SOURCE_EXTENSION: - continue - if self.compile: - bytecode_files.append(importlib.util.cache_from_source( - py_file, optimization='')) - if self.optimize > 0: - bytecode_files.append(importlib.util.cache_from_source( - py_file, optimization=self.optimize)) - - return bytecode_files - - - # -- External interface -------------------------------------------- - # (called by outsiders) - - def get_outputs(self): - """Return the list of files that would be installed if this command - were actually run. Not affected by the "dry-run" flag or whether - modules have actually been built yet. - """ - pure_outputs = \ - self._mutate_outputs(self.distribution.has_pure_modules(), - 'build_py', 'build_lib', - self.install_dir) - if self.compile: - bytecode_outputs = self._bytecode_filenames(pure_outputs) - else: - bytecode_outputs = [] - - ext_outputs = \ - self._mutate_outputs(self.distribution.has_ext_modules(), - 'build_ext', 'build_lib', - self.install_dir) - - return pure_outputs + bytecode_outputs + ext_outputs - - def get_inputs(self): - """Get the list of files that are input to this command, ie. the - files that get installed as they are named in the build tree. - The files in this list correspond one-to-one to the output - filenames returned by 'get_outputs()'. - """ - inputs = [] - - if self.distribution.has_pure_modules(): - build_py = self.get_finalized_command('build_py') - inputs.extend(build_py.get_outputs()) - - if self.distribution.has_ext_modules(): - build_ext = self.get_finalized_command('build_ext') - inputs.extend(build_ext.get_outputs()) - - return inputs diff --git a/Lib/distutils/command/install_scripts.py b/Lib/distutils/command/install_scripts.py deleted file mode 100644 index 31a1130ee54..00000000000 --- a/Lib/distutils/command/install_scripts.py +++ /dev/null @@ -1,60 +0,0 @@ -"""distutils.command.install_scripts - -Implements the Distutils 'install_scripts' command, for installing -Python scripts.""" - -# contributed by Bastian Kleineidam - -import os -from distutils.core import Command -from distutils import log -from stat import ST_MODE - - -class install_scripts(Command): - - description = "install scripts (Python or otherwise)" - - user_options = [ - ('install-dir=', 'd', "directory to install scripts to"), - ('build-dir=','b', "build directory (where to install from)"), - ('force', 'f', "force installation (overwrite existing files)"), - ('skip-build', None, "skip the build steps"), - ] - - boolean_options = ['force', 'skip-build'] - - def initialize_options(self): - self.install_dir = None - self.force = 0 - self.build_dir = None - self.skip_build = None - - def finalize_options(self): - self.set_undefined_options('build', ('build_scripts', 'build_dir')) - self.set_undefined_options('install', - ('install_scripts', 'install_dir'), - ('force', 'force'), - ('skip_build', 'skip_build'), - ) - - def run(self): - if not self.skip_build: - self.run_command('build_scripts') - self.outfiles = self.copy_tree(self.build_dir, self.install_dir) - if os.name == 'posix': - # Set the executable bits (owner, group, and world) on - # all the scripts we just installed. - for file in self.get_outputs(): - if self.dry_run: - log.info("changing mode of %s", file) - else: - mode = ((os.stat(file)[ST_MODE]) | 0o555) & 0o7777 - log.info("changing mode of %s to %o", file, mode) - os.chmod(file, mode) - - def get_inputs(self): - return self.distribution.scripts or [] - - def get_outputs(self): - return self.outfiles or [] diff --git a/Lib/distutils/command/register.py b/Lib/distutils/command/register.py deleted file mode 100644 index 0fac94e9e54..00000000000 --- a/Lib/distutils/command/register.py +++ /dev/null @@ -1,304 +0,0 @@ -"""distutils.command.register - -Implements the Distutils 'register' command (register with the repository). -""" - -# created 2002/10/21, Richard Jones - -import getpass -import io -import urllib.parse, urllib.request -from warnings import warn - -from distutils.core import PyPIRCCommand -from distutils.errors import * -from distutils import log - -class register(PyPIRCCommand): - - description = ("register the distribution with the Python package index") - user_options = PyPIRCCommand.user_options + [ - ('list-classifiers', None, - 'list the valid Trove classifiers'), - ('strict', None , - 'Will stop the registering if the meta-data are not fully compliant') - ] - boolean_options = PyPIRCCommand.boolean_options + [ - 'verify', 'list-classifiers', 'strict'] - - sub_commands = [('check', lambda self: True)] - - def initialize_options(self): - PyPIRCCommand.initialize_options(self) - self.list_classifiers = 0 - self.strict = 0 - - def finalize_options(self): - PyPIRCCommand.finalize_options(self) - # setting options for the `check` subcommand - check_options = {'strict': ('register', self.strict), - 'restructuredtext': ('register', 1)} - self.distribution.command_options['check'] = check_options - - def run(self): - self.finalize_options() - self._set_config() - - # Run sub commands - for cmd_name in self.get_sub_commands(): - self.run_command(cmd_name) - - if self.dry_run: - self.verify_metadata() - elif self.list_classifiers: - self.classifiers() - else: - self.send_metadata() - - def check_metadata(self): - """Deprecated API.""" - warn("distutils.command.register.check_metadata is deprecated, \ - use the check command instead", PendingDeprecationWarning) - check = self.distribution.get_command_obj('check') - check.ensure_finalized() - check.strict = self.strict - check.restructuredtext = 1 - check.run() - - def _set_config(self): - ''' Reads the configuration file and set attributes. - ''' - config = self._read_pypirc() - if config != {}: - self.username = config['username'] - self.password = config['password'] - self.repository = config['repository'] - self.realm = config['realm'] - self.has_config = True - else: - if self.repository not in ('pypi', self.DEFAULT_REPOSITORY): - raise ValueError('%s not found in .pypirc' % self.repository) - if self.repository == 'pypi': - self.repository = self.DEFAULT_REPOSITORY - self.has_config = False - - def classifiers(self): - ''' Fetch the list of classifiers from the server. - ''' - url = self.repository+'?:action=list_classifiers' - response = urllib.request.urlopen(url) - log.info(self._read_pypi_response(response)) - - def verify_metadata(self): - ''' Send the metadata to the package index server to be checked. - ''' - # send the info to the server and report the result - (code, result) = self.post_to_server(self.build_post_data('verify')) - log.info('Server response (%s): %s', code, result) - - def send_metadata(self): - ''' Send the metadata to the package index server. - - Well, do the following: - 1. figure who the user is, and then - 2. send the data as a Basic auth'ed POST. - - First we try to read the username/password from $HOME/.pypirc, - which is a ConfigParser-formatted file with a section - [distutils] containing username and password entries (both - in clear text). Eg: - - [distutils] - index-servers = - pypi - - [pypi] - username: fred - password: sekrit - - Otherwise, to figure who the user is, we offer the user three - choices: - - 1. use existing login, - 2. register as a new user, or - 3. set the password to a random string and email the user. - - ''' - # see if we can short-cut and get the username/password from the - # config - if self.has_config: - choice = '1' - username = self.username - password = self.password - else: - choice = 'x' - username = password = '' - - # get the user's login info - choices = '1 2 3 4'.split() - while choice not in choices: - self.announce('''\ -We need to know who you are, so please choose either: - 1. use your existing login, - 2. register as a new user, - 3. have the server generate a new password for you (and email it to you), or - 4. quit -Your selection [default 1]: ''', log.INFO) - choice = input() - if not choice: - choice = '1' - elif choice not in choices: - print('Please choose one of the four options!') - - if choice == '1': - # get the username and password - while not username: - username = input('Username: ') - while not password: - password = getpass.getpass('Password: ') - - # set up the authentication - auth = urllib.request.HTTPPasswordMgr() - host = urllib.parse.urlparse(self.repository)[1] - auth.add_password(self.realm, host, username, password) - # send the info to the server and report the result - code, result = self.post_to_server(self.build_post_data('submit'), - auth) - self.announce('Server response (%s): %s' % (code, result), - log.INFO) - - # possibly save the login - if code == 200: - if self.has_config: - # sharing the password in the distribution instance - # so the upload command can reuse it - self.distribution.password = password - else: - self.announce(('I can store your PyPI login so future ' - 'submissions will be faster.'), log.INFO) - self.announce('(the login will be stored in %s)' % \ - self._get_rc_file(), log.INFO) - choice = 'X' - while choice.lower() not in 'yn': - choice = input('Save your login (y/N)?') - if not choice: - choice = 'n' - if choice.lower() == 'y': - self._store_pypirc(username, password) - - elif choice == '2': - data = {':action': 'user'} - data['name'] = data['password'] = data['email'] = '' - data['confirm'] = None - while not data['name']: - data['name'] = input('Username: ') - while data['password'] != data['confirm']: - while not data['password']: - data['password'] = getpass.getpass('Password: ') - while not data['confirm']: - data['confirm'] = getpass.getpass(' Confirm: ') - if data['password'] != data['confirm']: - data['password'] = '' - data['confirm'] = None - print("Password and confirm don't match!") - while not data['email']: - data['email'] = input(' EMail: ') - code, result = self.post_to_server(data) - if code != 200: - log.info('Server response (%s): %s', code, result) - else: - log.info('You will receive an email shortly.') - log.info(('Follow the instructions in it to ' - 'complete registration.')) - elif choice == '3': - data = {':action': 'password_reset'} - data['email'] = '' - while not data['email']: - data['email'] = input('Your email address: ') - code, result = self.post_to_server(data) - log.info('Server response (%s): %s', code, result) - - def build_post_data(self, action): - # figure the data to send - the metadata plus some additional - # information used by the package server - meta = self.distribution.metadata - data = { - ':action': action, - 'metadata_version' : '1.0', - 'name': meta.get_name(), - 'version': meta.get_version(), - 'summary': meta.get_description(), - 'home_page': meta.get_url(), - 'author': meta.get_contact(), - 'author_email': meta.get_contact_email(), - 'license': meta.get_licence(), - 'description': meta.get_long_description(), - 'keywords': meta.get_keywords(), - 'platform': meta.get_platforms(), - 'classifiers': meta.get_classifiers(), - 'download_url': meta.get_download_url(), - # PEP 314 - 'provides': meta.get_provides(), - 'requires': meta.get_requires(), - 'obsoletes': meta.get_obsoletes(), - } - if data['provides'] or data['requires'] or data['obsoletes']: - data['metadata_version'] = '1.1' - return data - - def post_to_server(self, data, auth=None): - ''' Post a query to the server, and return a string response. - ''' - if 'name' in data: - self.announce('Registering %s to %s' % (data['name'], - self.repository), - log.INFO) - # Build up the MIME payload for the urllib2 POST data - boundary = '--------------GHSKFJDLGDS7543FJKLFHRE75642756743254' - sep_boundary = '\n--' + boundary - end_boundary = sep_boundary + '--' - body = io.StringIO() - for key, value in data.items(): - # handle multiple entries for the same name - if type(value) not in (type([]), type( () )): - value = [value] - for value in value: - value = str(value) - body.write(sep_boundary) - body.write('\nContent-Disposition: form-data; name="%s"'%key) - body.write("\n\n") - body.write(value) - if value and value[-1] == '\r': - body.write('\n') # write an extra newline (lurve Macs) - body.write(end_boundary) - body.write("\n") - body = body.getvalue().encode("utf-8") - - # build the Request - headers = { - 'Content-type': 'multipart/form-data; boundary=%s; charset=utf-8'%boundary, - 'Content-length': str(len(body)) - } - req = urllib.request.Request(self.repository, body, headers) - - # handle HTTP and include the Basic Auth handler - opener = urllib.request.build_opener( - urllib.request.HTTPBasicAuthHandler(password_mgr=auth) - ) - data = '' - try: - result = opener.open(req) - except urllib.error.HTTPError as e: - if self.show_response: - data = e.fp.read() - result = e.code, e.msg - except urllib.error.URLError as e: - result = 500, str(e) - else: - if self.show_response: - data = self._read_pypi_response(result) - result = 200, 'OK' - if self.show_response: - msg = '\n'.join(('-' * 75, data, '-' * 75)) - self.announce(msg, log.INFO) - return result diff --git a/Lib/distutils/command/sdist.py b/Lib/distutils/command/sdist.py deleted file mode 100644 index 4fd1d4715de..00000000000 --- a/Lib/distutils/command/sdist.py +++ /dev/null @@ -1,456 +0,0 @@ -"""distutils.command.sdist - -Implements the Distutils 'sdist' command (create a source distribution).""" - -import os -import sys -from types import * -from glob import glob -from warnings import warn - -from distutils.core import Command -from distutils import dir_util, dep_util, file_util, archive_util -from distutils.text_file import TextFile -from distutils.errors import * -from distutils.filelist import FileList -from distutils import log -from distutils.util import convert_path - -def show_formats(): - """Print all possible values for the 'formats' option (used by - the "--help-formats" command-line option). - """ - from distutils.fancy_getopt import FancyGetopt - from distutils.archive_util import ARCHIVE_FORMATS - formats = [] - for format in ARCHIVE_FORMATS.keys(): - formats.append(("formats=" + format, None, - ARCHIVE_FORMATS[format][2])) - formats.sort() - FancyGetopt(formats).print_help( - "List of available source distribution formats:") - -class sdist(Command): - - description = "create a source distribution (tarball, zip file, etc.)" - - def checking_metadata(self): - """Callable used for the check sub-command. - - Placed here so user_options can view it""" - return self.metadata_check - - user_options = [ - ('template=', 't', - "name of manifest template file [default: MANIFEST.in]"), - ('manifest=', 'm', - "name of manifest file [default: MANIFEST]"), - ('use-defaults', None, - "include the default file set in the manifest " - "[default; disable with --no-defaults]"), - ('no-defaults', None, - "don't include the default file set"), - ('prune', None, - "specifically exclude files/directories that should not be " - "distributed (build tree, RCS/CVS dirs, etc.) " - "[default; disable with --no-prune]"), - ('no-prune', None, - "don't automatically exclude anything"), - ('manifest-only', 'o', - "just regenerate the manifest and then stop " - "(implies --force-manifest)"), - ('force-manifest', 'f', - "forcibly regenerate the manifest and carry on as usual. " - "Deprecated: now the manifest is always regenerated."), - ('formats=', None, - "formats for source distribution (comma-separated list)"), - ('keep-temp', 'k', - "keep the distribution tree around after creating " + - "archive file(s)"), - ('dist-dir=', 'd', - "directory to put the source distribution archive(s) in " - "[default: dist]"), - ('metadata-check', None, - "Ensure that all required elements of meta-data " - "are supplied. Warn if any missing. [default]"), - ('owner=', 'u', - "Owner name used when creating a tar file [default: current user]"), - ('group=', 'g', - "Group name used when creating a tar file [default: current group]"), - ] - - boolean_options = ['use-defaults', 'prune', - 'manifest-only', 'force-manifest', - 'keep-temp', 'metadata-check'] - - help_options = [ - ('help-formats', None, - "list available distribution formats", show_formats), - ] - - negative_opt = {'no-defaults': 'use-defaults', - 'no-prune': 'prune' } - - sub_commands = [('check', checking_metadata)] - - def initialize_options(self): - # 'template' and 'manifest' are, respectively, the names of - # the manifest template and manifest file. - self.template = None - self.manifest = None - - # 'use_defaults': if true, we will include the default file set - # in the manifest - self.use_defaults = 1 - self.prune = 1 - - self.manifest_only = 0 - self.force_manifest = 0 - - self.formats = ['gztar'] - self.keep_temp = 0 - self.dist_dir = None - - self.archive_files = None - self.metadata_check = 1 - self.owner = None - self.group = None - - def finalize_options(self): - if self.manifest is None: - self.manifest = "MANIFEST" - if self.template is None: - self.template = "MANIFEST.in" - - self.ensure_string_list('formats') - - bad_format = archive_util.check_archive_formats(self.formats) - if bad_format: - raise DistutilsOptionError( - "unknown archive format '%s'" % bad_format) - - if self.dist_dir is None: - self.dist_dir = "dist" - - def run(self): - # 'filelist' contains the list of files that will make up the - # manifest - self.filelist = FileList() - - # Run sub commands - for cmd_name in self.get_sub_commands(): - self.run_command(cmd_name) - - # Do whatever it takes to get the list of files to process - # (process the manifest template, read an existing manifest, - # whatever). File list is accumulated in 'self.filelist'. - self.get_file_list() - - # If user just wanted us to regenerate the manifest, stop now. - if self.manifest_only: - return - - # Otherwise, go ahead and create the source distribution tarball, - # or zipfile, or whatever. - self.make_distribution() - - def check_metadata(self): - """Deprecated API.""" - warn("distutils.command.sdist.check_metadata is deprecated, \ - use the check command instead", PendingDeprecationWarning) - check = self.distribution.get_command_obj('check') - check.ensure_finalized() - check.run() - - def get_file_list(self): - """Figure out the list of files to include in the source - distribution, and put it in 'self.filelist'. This might involve - reading the manifest template (and writing the manifest), or just - reading the manifest, or just using the default file set -- it all - depends on the user's options. - """ - # new behavior when using a template: - # the file list is recalculated every time because - # even if MANIFEST.in or setup.py are not changed - # the user might have added some files in the tree that - # need to be included. - # - # This makes --force the default and only behavior with templates. - template_exists = os.path.isfile(self.template) - if not template_exists and self._manifest_is_not_generated(): - self.read_manifest() - self.filelist.sort() - self.filelist.remove_duplicates() - return - - if not template_exists: - self.warn(("manifest template '%s' does not exist " + - "(using default file list)") % - self.template) - self.filelist.findall() - - if self.use_defaults: - self.add_defaults() - - if template_exists: - self.read_template() - - if self.prune: - self.prune_file_list() - - self.filelist.sort() - self.filelist.remove_duplicates() - self.write_manifest() - - def add_defaults(self): - """Add all the default files to self.filelist: - - README or README.txt - - setup.py - - test/test*.py - - all pure Python modules mentioned in setup script - - all files pointed by package_data (build_py) - - all files defined in data_files. - - all files defined as scripts. - - all C sources listed as part of extensions or C libraries - in the setup script (doesn't catch C headers!) - Warns if (README or README.txt) or setup.py are missing; everything - else is optional. - """ - standards = [('README', 'README.txt'), self.distribution.script_name] - for fn in standards: - if isinstance(fn, tuple): - alts = fn - got_it = False - for fn in alts: - if os.path.exists(fn): - got_it = True - self.filelist.append(fn) - break - - if not got_it: - self.warn("standard file not found: should have one of " + - ', '.join(alts)) - else: - if os.path.exists(fn): - self.filelist.append(fn) - else: - self.warn("standard file '%s' not found" % fn) - - optional = ['test/test*.py', 'setup.cfg'] - for pattern in optional: - files = filter(os.path.isfile, glob(pattern)) - self.filelist.extend(files) - - # build_py is used to get: - # - python modules - # - files defined in package_data - build_py = self.get_finalized_command('build_py') - - # getting python files - if self.distribution.has_pure_modules(): - self.filelist.extend(build_py.get_source_files()) - - # getting package_data files - # (computed in build_py.data_files by build_py.finalize_options) - for pkg, src_dir, build_dir, filenames in build_py.data_files: - for filename in filenames: - self.filelist.append(os.path.join(src_dir, filename)) - - # getting distribution.data_files - if self.distribution.has_data_files(): - for item in self.distribution.data_files: - if isinstance(item, str): # plain file - item = convert_path(item) - if os.path.isfile(item): - self.filelist.append(item) - else: # a (dirname, filenames) tuple - dirname, filenames = item - for f in filenames: - f = convert_path(f) - if os.path.isfile(f): - self.filelist.append(f) - - if self.distribution.has_ext_modules(): - build_ext = self.get_finalized_command('build_ext') - self.filelist.extend(build_ext.get_source_files()) - - if self.distribution.has_c_libraries(): - build_clib = self.get_finalized_command('build_clib') - self.filelist.extend(build_clib.get_source_files()) - - if self.distribution.has_scripts(): - build_scripts = self.get_finalized_command('build_scripts') - self.filelist.extend(build_scripts.get_source_files()) - - def read_template(self): - """Read and parse manifest template file named by self.template. - - (usually "MANIFEST.in") The parsing and processing is done by - 'self.filelist', which updates itself accordingly. - """ - log.info("reading manifest template '%s'", self.template) - template = TextFile(self.template, strip_comments=1, skip_blanks=1, - join_lines=1, lstrip_ws=1, rstrip_ws=1, - collapse_join=1) - - try: - while True: - line = template.readline() - if line is None: # end of file - break - - try: - self.filelist.process_template_line(line) - # the call above can raise a DistutilsTemplateError for - # malformed lines, or a ValueError from the lower-level - # convert_path function - except (DistutilsTemplateError, ValueError) as msg: - self.warn("%s, line %d: %s" % (template.filename, - template.current_line, - msg)) - finally: - template.close() - - def prune_file_list(self): - """Prune off branches that might slip into the file list as created - by 'read_template()', but really don't belong there: - * the build tree (typically "build") - * the release tree itself (only an issue if we ran "sdist" - previously with --keep-temp, or it aborted) - * any RCS, CVS, .svn, .hg, .git, .bzr, _darcs directories - """ - build = self.get_finalized_command('build') - base_dir = self.distribution.get_fullname() - - self.filelist.exclude_pattern(None, prefix=build.build_base) - self.filelist.exclude_pattern(None, prefix=base_dir) - - if sys.platform == 'win32': - seps = r'/|\\' - else: - seps = '/' - - vcs_dirs = ['RCS', 'CVS', r'\.svn', r'\.hg', r'\.git', r'\.bzr', - '_darcs'] - vcs_ptrn = r'(^|%s)(%s)(%s).*' % (seps, '|'.join(vcs_dirs), seps) - self.filelist.exclude_pattern(vcs_ptrn, is_regex=1) - - def write_manifest(self): - """Write the file list in 'self.filelist' (presumably as filled in - by 'add_defaults()' and 'read_template()') to the manifest file - named by 'self.manifest'. - """ - if self._manifest_is_not_generated(): - log.info("not writing to manually maintained " - "manifest file '%s'" % self.manifest) - return - - content = self.filelist.files[:] - content.insert(0, '# file GENERATED by distutils, do NOT edit') - self.execute(file_util.write_file, (self.manifest, content), - "writing manifest file '%s'" % self.manifest) - - def _manifest_is_not_generated(self): - # check for special comment used in 3.1.3 and higher - if not os.path.isfile(self.manifest): - return False - - fp = open(self.manifest) - try: - first_line = fp.readline() - finally: - fp.close() - return first_line != '# file GENERATED by distutils, do NOT edit\n' - - def read_manifest(self): - """Read the manifest file (named by 'self.manifest') and use it to - fill in 'self.filelist', the list of files to include in the source - distribution. - """ - log.info("reading manifest file '%s'", self.manifest) - manifest = open(self.manifest) - for line in manifest: - # ignore comments and blank lines - line = line.strip() - if line.startswith('#') or not line: - continue - self.filelist.append(line) - manifest.close() - - def make_release_tree(self, base_dir, files): - """Create the directory tree that will become the source - distribution archive. All directories implied by the filenames in - 'files' are created under 'base_dir', and then we hard link or copy - (if hard linking is unavailable) those files into place. - Essentially, this duplicates the developer's source tree, but in a - directory named after the distribution, containing only the files - to be distributed. - """ - # Create all the directories under 'base_dir' necessary to - # put 'files' there; the 'mkpath()' is just so we don't die - # if the manifest happens to be empty. - self.mkpath(base_dir) - dir_util.create_tree(base_dir, files, dry_run=self.dry_run) - - # And walk over the list of files, either making a hard link (if - # os.link exists) to each one that doesn't already exist in its - # corresponding location under 'base_dir', or copying each file - # that's out-of-date in 'base_dir'. (Usually, all files will be - # out-of-date, because by default we blow away 'base_dir' when - # we're done making the distribution archives.) - - if hasattr(os, 'link'): # can make hard links on this system - link = 'hard' - msg = "making hard links in %s..." % base_dir - else: # nope, have to copy - link = None - msg = "copying files to %s..." % base_dir - - if not files: - log.warn("no files to distribute -- empty manifest?") - else: - log.info(msg) - for file in files: - if not os.path.isfile(file): - log.warn("'%s' not a regular file -- skipping", file) - else: - dest = os.path.join(base_dir, file) - self.copy_file(file, dest, link=link) - - self.distribution.metadata.write_pkg_info(base_dir) - - def make_distribution(self): - """Create the source distribution(s). First, we create the release - tree with 'make_release_tree()'; then, we create all required - archive files (according to 'self.formats') from the release tree. - Finally, we clean up by blowing away the release tree (unless - 'self.keep_temp' is true). The list of archive files created is - stored so it can be retrieved later by 'get_archive_files()'. - """ - # Don't warn about missing meta-data here -- should be (and is!) - # done elsewhere. - base_dir = self.distribution.get_fullname() - base_name = os.path.join(self.dist_dir, base_dir) - - self.make_release_tree(base_dir, self.filelist.files) - archive_files = [] # remember names of files we create - # tar archive must be created last to avoid overwrite and remove - if 'tar' in self.formats: - self.formats.append(self.formats.pop(self.formats.index('tar'))) - - for fmt in self.formats: - file = self.make_archive(base_name, fmt, base_dir=base_dir, - owner=self.owner, group=self.group) - archive_files.append(file) - self.distribution.dist_files.append(('sdist', '', file)) - - self.archive_files = archive_files - - if not self.keep_temp: - dir_util.remove_tree(base_dir, dry_run=self.dry_run) - - def get_archive_files(self): - """Return the list of archive files created when the command - was run, or None if the command hasn't run yet. - """ - return self.archive_files diff --git a/Lib/distutils/command/upload.py b/Lib/distutils/command/upload.py deleted file mode 100644 index 32dda359bad..00000000000 --- a/Lib/distutils/command/upload.py +++ /dev/null @@ -1,200 +0,0 @@ -""" -distutils.command.upload - -Implements the Distutils 'upload' subcommand (upload package to a package -index). -""" - -import os -import io -import platform -import hashlib -from base64 import standard_b64encode -from urllib.request import urlopen, Request, HTTPError -from urllib.parse import urlparse -from distutils.errors import DistutilsError, DistutilsOptionError -from distutils.core import PyPIRCCommand -from distutils.spawn import spawn -from distutils import log - -class upload(PyPIRCCommand): - - description = "upload binary package to PyPI" - - user_options = PyPIRCCommand.user_options + [ - ('sign', 's', - 'sign files to upload using gpg'), - ('identity=', 'i', 'GPG identity used to sign files'), - ] - - boolean_options = PyPIRCCommand.boolean_options + ['sign'] - - def initialize_options(self): - PyPIRCCommand.initialize_options(self) - self.username = '' - self.password = '' - self.show_response = 0 - self.sign = False - self.identity = None - - def finalize_options(self): - PyPIRCCommand.finalize_options(self) - if self.identity and not self.sign: - raise DistutilsOptionError( - "Must use --sign for --identity to have meaning" - ) - config = self._read_pypirc() - if config != {}: - self.username = config['username'] - self.password = config['password'] - self.repository = config['repository'] - self.realm = config['realm'] - - # getting the password from the distribution - # if previously set by the register command - if not self.password and self.distribution.password: - self.password = self.distribution.password - - def run(self): - if not self.distribution.dist_files: - msg = ("Must create and upload files in one command " - "(e.g. setup.py sdist upload)") - raise DistutilsOptionError(msg) - for command, pyversion, filename in self.distribution.dist_files: - self.upload_file(command, pyversion, filename) - - def upload_file(self, command, pyversion, filename): - # Makes sure the repository URL is compliant - schema, netloc, url, params, query, fragments = \ - urlparse(self.repository) - if params or query or fragments: - raise AssertionError("Incompatible url %s" % self.repository) - - if schema not in ('http', 'https'): - raise AssertionError("unsupported schema " + schema) - - # Sign if requested - if self.sign: - gpg_args = ["gpg", "--detach-sign", "-a", filename] - if self.identity: - gpg_args[2:2] = ["--local-user", self.identity] - spawn(gpg_args, - dry_run=self.dry_run) - - # Fill in the data - send all the meta-data in case we need to - # register a new release - f = open(filename,'rb') - try: - content = f.read() - finally: - f.close() - meta = self.distribution.metadata - data = { - # action - ':action': 'file_upload', - 'protocol_version': '1', - - # identify release - 'name': meta.get_name(), - 'version': meta.get_version(), - - # file content - 'content': (os.path.basename(filename),content), - 'filetype': command, - 'pyversion': pyversion, - 'md5_digest': hashlib.md5(content).hexdigest(), - - # additional meta-data - 'metadata_version': '1.0', - 'summary': meta.get_description(), - 'home_page': meta.get_url(), - 'author': meta.get_contact(), - 'author_email': meta.get_contact_email(), - 'license': meta.get_licence(), - 'description': meta.get_long_description(), - 'keywords': meta.get_keywords(), - 'platform': meta.get_platforms(), - 'classifiers': meta.get_classifiers(), - 'download_url': meta.get_download_url(), - # PEP 314 - 'provides': meta.get_provides(), - 'requires': meta.get_requires(), - 'obsoletes': meta.get_obsoletes(), - } - comment = '' - if command == 'bdist_rpm': - dist, version, id = platform.dist() - if dist: - comment = 'built for %s %s' % (dist, version) - elif command == 'bdist_dumb': - comment = 'built for %s' % platform.platform(terse=1) - data['comment'] = comment - - if self.sign: - data['gpg_signature'] = (os.path.basename(filename) + ".asc", - open(filename+".asc", "rb").read()) - - # set up the authentication - user_pass = (self.username + ":" + self.password).encode('ascii') - # The exact encoding of the authentication string is debated. - # Anyway PyPI only accepts ascii for both username or password. - auth = "Basic " + standard_b64encode(user_pass).decode('ascii') - - # Build up the MIME payload for the POST data - boundary = '--------------GHSKFJDLGDS7543FJKLFHRE75642756743254' - sep_boundary = b'\r\n--' + boundary.encode('ascii') - end_boundary = sep_boundary + b'--\r\n' - body = io.BytesIO() - for key, value in data.items(): - title = '\r\nContent-Disposition: form-data; name="%s"' % key - # handle multiple entries for the same name - if not isinstance(value, list): - value = [value] - for value in value: - if type(value) is tuple: - title += '; filename="%s"' % value[0] - value = value[1] - else: - value = str(value).encode('utf-8') - body.write(sep_boundary) - body.write(title.encode('utf-8')) - body.write(b"\r\n\r\n") - body.write(value) - body.write(end_boundary) - body = body.getvalue() - - msg = "Submitting %s to %s" % (filename, self.repository) - self.announce(msg, log.INFO) - - # build the Request - headers = { - 'Content-type': 'multipart/form-data; boundary=%s' % boundary, - 'Content-length': str(len(body)), - 'Authorization': auth, - } - - request = Request(self.repository, data=body, - headers=headers) - # send the data - try: - result = urlopen(request) - status = result.getcode() - reason = result.msg - except HTTPError as e: - status = e.code - reason = e.msg - except OSError as e: - self.announce(str(e), log.ERROR) - raise - - if status == 200: - self.announce('Server response (%s): %s' % (status, reason), - log.INFO) - if self.show_response: - text = self._read_pypi_response(result) - msg = '\n'.join(('-' * 75, text, '-' * 75)) - self.announce(msg, log.INFO) - else: - msg = 'Upload failed (%s): %s' % (status, reason) - self.announce(msg, log.ERROR) - raise DistutilsError(msg) diff --git a/Lib/distutils/config.py b/Lib/distutils/config.py deleted file mode 100644 index bf8d8dd2f5a..00000000000 --- a/Lib/distutils/config.py +++ /dev/null @@ -1,131 +0,0 @@ -"""distutils.pypirc - -Provides the PyPIRCCommand class, the base class for the command classes -that uses .pypirc in the distutils.command package. -""" -import os -from configparser import RawConfigParser - -from distutils.cmd import Command - -DEFAULT_PYPIRC = """\ -[distutils] -index-servers = - pypi - -[pypi] -username:%s -password:%s -""" - -class PyPIRCCommand(Command): - """Base command that knows how to handle the .pypirc file - """ - DEFAULT_REPOSITORY = 'https://upload.pypi.org/legacy/' - DEFAULT_REALM = 'pypi' - repository = None - realm = None - - user_options = [ - ('repository=', 'r', - "url of repository [default: %s]" % \ - DEFAULT_REPOSITORY), - ('show-response', None, - 'display full response text from server')] - - boolean_options = ['show-response'] - - def _get_rc_file(self): - """Returns rc file path.""" - return os.path.join(os.path.expanduser('~'), '.pypirc') - - def _store_pypirc(self, username, password): - """Creates a default .pypirc file.""" - rc = self._get_rc_file() - with os.fdopen(os.open(rc, os.O_CREAT | os.O_WRONLY, 0o600), 'w') as f: - f.write(DEFAULT_PYPIRC % (username, password)) - - def _read_pypirc(self): - """Reads the .pypirc file.""" - rc = self._get_rc_file() - if os.path.exists(rc): - self.announce('Using PyPI login from %s' % rc) - repository = self.repository or self.DEFAULT_REPOSITORY - realm = self.realm or self.DEFAULT_REALM - - config = RawConfigParser() - config.read(rc) - sections = config.sections() - if 'distutils' in sections: - # let's get the list of servers - index_servers = config.get('distutils', 'index-servers') - _servers = [server.strip() for server in - index_servers.split('\n') - if server.strip() != ''] - if _servers == []: - # nothing set, let's try to get the default pypi - if 'pypi' in sections: - _servers = ['pypi'] - else: - # the file is not properly defined, returning - # an empty dict - return {} - for server in _servers: - current = {'server': server} - current['username'] = config.get(server, 'username') - - # optional params - for key, default in (('repository', - self.DEFAULT_REPOSITORY), - ('realm', self.DEFAULT_REALM), - ('password', None)): - if config.has_option(server, key): - current[key] = config.get(server, key) - else: - current[key] = default - - # work around people having "repository" for the "pypi" - # section of their config set to the HTTP (rather than - # HTTPS) URL - if (server == 'pypi' and - repository in (self.DEFAULT_REPOSITORY, 'pypi')): - current['repository'] = self.DEFAULT_REPOSITORY - return current - - if (current['server'] == repository or - current['repository'] == repository): - return current - elif 'server-login' in sections: - # old format - server = 'server-login' - if config.has_option(server, 'repository'): - repository = config.get(server, 'repository') - else: - repository = self.DEFAULT_REPOSITORY - return {'username': config.get(server, 'username'), - 'password': config.get(server, 'password'), - 'repository': repository, - 'server': server, - 'realm': self.DEFAULT_REALM} - - return {} - - def _read_pypi_response(self, response): - """Read and decode a PyPI HTTP response.""" - import cgi - content_type = response.getheader('content-type', 'text/plain') - encoding = cgi.parse_header(content_type)[1].get('charset', 'ascii') - return response.read().decode(encoding) - - def initialize_options(self): - """Initialize options.""" - self.repository = None - self.realm = None - self.show_response = 0 - - def finalize_options(self): - """Finalizes options.""" - if self.repository is None: - self.repository = self.DEFAULT_REPOSITORY - if self.realm is None: - self.realm = self.DEFAULT_REALM diff --git a/Lib/distutils/core.py b/Lib/distutils/core.py deleted file mode 100644 index d603d4a45a7..00000000000 --- a/Lib/distutils/core.py +++ /dev/null @@ -1,234 +0,0 @@ -"""distutils.core - -The only module that needs to be imported to use the Distutils; provides -the 'setup' function (which is to be called from the setup script). Also -indirectly provides the Distribution and Command classes, although they are -really defined in distutils.dist and distutils.cmd. -""" - -import os -import sys - -from distutils.debug import DEBUG -from distutils.errors import * - -# Mainly import these so setup scripts can "from distutils.core import" them. -from distutils.dist import Distribution -from distutils.cmd import Command -from distutils.config import PyPIRCCommand -from distutils.extension import Extension - -# This is a barebones help message generated displayed when the user -# runs the setup script with no arguments at all. More useful help -# is generated with various --help options: global help, list commands, -# and per-command help. -USAGE = """\ -usage: %(script)s [global_opts] cmd1 [cmd1_opts] [cmd2 [cmd2_opts] ...] - or: %(script)s --help [cmd1 cmd2 ...] - or: %(script)s --help-commands - or: %(script)s cmd --help -""" - -def gen_usage (script_name): - script = os.path.basename(script_name) - return USAGE % vars() - - -# Some mild magic to control the behaviour of 'setup()' from 'run_setup()'. -_setup_stop_after = None -_setup_distribution = None - -# Legal keyword arguments for the setup() function -setup_keywords = ('distclass', 'script_name', 'script_args', 'options', - 'name', 'version', 'author', 'author_email', - 'maintainer', 'maintainer_email', 'url', 'license', - 'description', 'long_description', 'keywords', - 'platforms', 'classifiers', 'download_url', - 'requires', 'provides', 'obsoletes', - ) - -# Legal keyword arguments for the Extension constructor -extension_keywords = ('name', 'sources', 'include_dirs', - 'define_macros', 'undef_macros', - 'library_dirs', 'libraries', 'runtime_library_dirs', - 'extra_objects', 'extra_compile_args', 'extra_link_args', - 'swig_opts', 'export_symbols', 'depends', 'language') - -def setup (**attrs): - """The gateway to the Distutils: do everything your setup script needs - to do, in a highly flexible and user-driven way. Briefly: create a - Distribution instance; find and parse config files; parse the command - line; run each Distutils command found there, customized by the options - supplied to 'setup()' (as keyword arguments), in config files, and on - the command line. - - The Distribution instance might be an instance of a class supplied via - the 'distclass' keyword argument to 'setup'; if no such class is - supplied, then the Distribution class (in dist.py) is instantiated. - All other arguments to 'setup' (except for 'cmdclass') are used to set - attributes of the Distribution instance. - - The 'cmdclass' argument, if supplied, is a dictionary mapping command - names to command classes. Each command encountered on the command line - will be turned into a command class, which is in turn instantiated; any - class found in 'cmdclass' is used in place of the default, which is - (for command 'foo_bar') class 'foo_bar' in module - 'distutils.command.foo_bar'. The command class must provide a - 'user_options' attribute which is a list of option specifiers for - 'distutils.fancy_getopt'. Any command-line options between the current - and the next command are used to set attributes of the current command - object. - - When the entire command-line has been successfully parsed, calls the - 'run()' method on each command object in turn. This method will be - driven entirely by the Distribution object (which each command object - has a reference to, thanks to its constructor), and the - command-specific options that became attributes of each command - object. - """ - - global _setup_stop_after, _setup_distribution - - # Determine the distribution class -- either caller-supplied or - # our Distribution (see below). - klass = attrs.get('distclass') - if klass: - del attrs['distclass'] - else: - klass = Distribution - - if 'script_name' not in attrs: - attrs['script_name'] = os.path.basename(sys.argv[0]) - if 'script_args' not in attrs: - attrs['script_args'] = sys.argv[1:] - - # Create the Distribution instance, using the remaining arguments - # (ie. everything except distclass) to initialize it - try: - _setup_distribution = dist = klass(attrs) - except DistutilsSetupError as msg: - if 'name' not in attrs: - raise SystemExit("error in setup command: %s" % msg) - else: - raise SystemExit("error in %s setup command: %s" % \ - (attrs['name'], msg)) - - if _setup_stop_after == "init": - return dist - - # Find and parse the config file(s): they will override options from - # the setup script, but be overridden by the command line. - dist.parse_config_files() - - if DEBUG: - print("options (after parsing config files):") - dist.dump_option_dicts() - - if _setup_stop_after == "config": - return dist - - # Parse the command line and override config files; any - # command-line errors are the end user's fault, so turn them into - # SystemExit to suppress tracebacks. - try: - ok = dist.parse_command_line() - except DistutilsArgError as msg: - raise SystemExit(gen_usage(dist.script_name) + "\nerror: %s" % msg) - - if DEBUG: - print("options (after parsing command line):") - dist.dump_option_dicts() - - if _setup_stop_after == "commandline": - return dist - - # And finally, run all the commands found on the command line. - if ok: - try: - dist.run_commands() - except KeyboardInterrupt: - raise SystemExit("interrupted") - except OSError as exc: - if DEBUG: - sys.stderr.write("error: %s\n" % (exc,)) - raise - else: - raise SystemExit("error: %s" % (exc,)) - - except (DistutilsError, - CCompilerError) as msg: - if DEBUG: - raise - else: - raise SystemExit("error: " + str(msg)) - - return dist - -# setup () - - -def run_setup (script_name, script_args=None, stop_after="run"): - """Run a setup script in a somewhat controlled environment, and - return the Distribution instance that drives things. This is useful - if you need to find out the distribution meta-data (passed as - keyword args from 'script' to 'setup()', or the contents of the - config files or command-line. - - 'script_name' is a file that will be read and run with 'exec()'; - 'sys.argv[0]' will be replaced with 'script' for the duration of the - call. 'script_args' is a list of strings; if supplied, - 'sys.argv[1:]' will be replaced by 'script_args' for the duration of - the call. - - 'stop_after' tells 'setup()' when to stop processing; possible - values: - init - stop after the Distribution instance has been created and - populated with the keyword arguments to 'setup()' - config - stop after config files have been parsed (and their data - stored in the Distribution instance) - commandline - stop after the command-line ('sys.argv[1:]' or 'script_args') - have been parsed (and the data stored in the Distribution) - run [default] - stop after all commands have been run (the same as if 'setup()' - had been called in the usual way - - Returns the Distribution instance, which provides all information - used to drive the Distutils. - """ - if stop_after not in ('init', 'config', 'commandline', 'run'): - raise ValueError("invalid value for 'stop_after': %r" % (stop_after,)) - - global _setup_stop_after, _setup_distribution - _setup_stop_after = stop_after - - save_argv = sys.argv.copy() - g = {'__file__': script_name} - try: - try: - sys.argv[0] = script_name - if script_args is not None: - sys.argv[1:] = script_args - with open(script_name, 'rb') as f: - exec(f.read(), g) - finally: - sys.argv = save_argv - _setup_stop_after = None - except SystemExit: - # Hmm, should we do something if exiting with a non-zero code - # (ie. error)? - pass - - if _setup_distribution is None: - raise RuntimeError(("'distutils.core.setup()' was never called -- " - "perhaps '%s' is not a Distutils setup script?") % \ - script_name) - - # I wonder if the setup script's namespace -- g and l -- would be of - # any interest to callers? - #print "_setup_distribution:", _setup_distribution - return _setup_distribution - -# run_setup () diff --git a/Lib/distutils/cygwinccompiler.py b/Lib/distutils/cygwinccompiler.py deleted file mode 100644 index 1c369903477..00000000000 --- a/Lib/distutils/cygwinccompiler.py +++ /dev/null @@ -1,405 +0,0 @@ -"""distutils.cygwinccompiler - -Provides the CygwinCCompiler class, a subclass of UnixCCompiler that -handles the Cygwin port of the GNU C compiler to Windows. It also contains -the Mingw32CCompiler class which handles the mingw32 port of GCC (same as -cygwin in no-cygwin mode). -""" - -# problems: -# -# * if you use a msvc compiled python version (1.5.2) -# 1. you have to insert a __GNUC__ section in its config.h -# 2. you have to generate an import library for its dll -# - create a def-file for python??.dll -# - create an import library using -# dlltool --dllname python15.dll --def python15.def \ -# --output-lib libpython15.a -# -# see also http://starship.python.net/crew/kernr/mingw32/Notes.html -# -# * We put export_symbols in a def-file, and don't use -# --export-all-symbols because it doesn't worked reliable in some -# tested configurations. And because other windows compilers also -# need their symbols specified this no serious problem. -# -# tested configurations: -# -# * cygwin gcc 2.91.57/ld 2.9.4/dllwrap 0.2.4 works -# (after patching python's config.h and for C++ some other include files) -# see also http://starship.python.net/crew/kernr/mingw32/Notes.html -# * mingw32 gcc 2.95.2/ld 2.9.4/dllwrap 0.2.4 works -# (ld doesn't support -shared, so we use dllwrap) -# * cygwin gcc 2.95.2/ld 2.10.90/dllwrap 2.10.90 works now -# - its dllwrap doesn't work, there is a bug in binutils 2.10.90 -# see also http://sources.redhat.com/ml/cygwin/2000-06/msg01274.html -# - using gcc -mdll instead dllwrap doesn't work without -static because -# it tries to link against dlls instead their import libraries. (If -# it finds the dll first.) -# By specifying -static we force ld to link against the import libraries, -# this is windows standard and there are normally not the necessary symbols -# in the dlls. -# *** only the version of June 2000 shows these problems -# * cygwin gcc 3.2/ld 2.13.90 works -# (ld supports -shared) -# * mingw gcc 3.2/ld 2.13 works -# (ld supports -shared) - -import os -import sys -import copy -from subprocess import Popen, PIPE, check_output -import re - -from distutils.ccompiler import gen_preprocess_options, gen_lib_options -from distutils.unixccompiler import UnixCCompiler -from distutils.file_util import write_file -from distutils.errors import (DistutilsExecError, CCompilerError, - CompileError, UnknownFileError) -from distutils import log -from distutils.version import LooseVersion -from distutils.spawn import find_executable - -def get_msvcr(): - """Include the appropriate MSVC runtime library if Python was built - with MSVC 7.0 or later. - """ - msc_pos = sys.version.find('MSC v.') - if msc_pos != -1: - msc_ver = sys.version[msc_pos+6:msc_pos+10] - if msc_ver == '1300': - # MSVC 7.0 - return ['msvcr70'] - elif msc_ver == '1310': - # MSVC 7.1 - return ['msvcr71'] - elif msc_ver == '1400': - # VS2005 / MSVC 8.0 - return ['msvcr80'] - elif msc_ver == '1500': - # VS2008 / MSVC 9.0 - return ['msvcr90'] - elif msc_ver == '1600': - # VS2010 / MSVC 10.0 - return ['msvcr100'] - else: - raise ValueError("Unknown MS Compiler version %s " % msc_ver) - - -class CygwinCCompiler(UnixCCompiler): - """ Handles the Cygwin port of the GNU C compiler to Windows. - """ - compiler_type = 'cygwin' - obj_extension = ".o" - static_lib_extension = ".a" - shared_lib_extension = ".dll" - static_lib_format = "lib%s%s" - shared_lib_format = "%s%s" - exe_extension = ".exe" - - def __init__(self, verbose=0, dry_run=0, force=0): - - UnixCCompiler.__init__(self, verbose, dry_run, force) - - status, details = check_config_h() - self.debug_print("Python's GCC status: %s (details: %s)" % - (status, details)) - if status is not CONFIG_H_OK: - self.warn( - "Python's pyconfig.h doesn't seem to support your compiler. " - "Reason: %s. " - "Compiling may fail because of undefined preprocessor macros." - % details) - - self.gcc_version, self.ld_version, self.dllwrap_version = \ - get_versions() - self.debug_print(self.compiler_type + ": gcc %s, ld %s, dllwrap %s\n" % - (self.gcc_version, - self.ld_version, - self.dllwrap_version) ) - - # ld_version >= "2.10.90" and < "2.13" should also be able to use - # gcc -mdll instead of dllwrap - # Older dllwraps had own version numbers, newer ones use the - # same as the rest of binutils ( also ld ) - # dllwrap 2.10.90 is buggy - if self.ld_version >= "2.10.90": - self.linker_dll = "gcc" - else: - self.linker_dll = "dllwrap" - - # ld_version >= "2.13" support -shared so use it instead of - # -mdll -static - if self.ld_version >= "2.13": - shared_option = "-shared" - else: - shared_option = "-mdll -static" - - # Hard-code GCC because that's what this is all about. - # XXX optimization, warnings etc. should be customizable. - self.set_executables(compiler='gcc -mcygwin -O -Wall', - compiler_so='gcc -mcygwin -mdll -O -Wall', - compiler_cxx='g++ -mcygwin -O -Wall', - linker_exe='gcc -mcygwin', - linker_so=('%s -mcygwin %s' % - (self.linker_dll, shared_option))) - - # cygwin and mingw32 need different sets of libraries - if self.gcc_version == "2.91.57": - # cygwin shouldn't need msvcrt, but without the dlls will crash - # (gcc version 2.91.57) -- perhaps something about initialization - self.dll_libraries=["msvcrt"] - self.warn( - "Consider upgrading to a newer version of gcc") - else: - # Include the appropriate MSVC runtime library if Python was built - # with MSVC 7.0 or later. - self.dll_libraries = get_msvcr() - - def _compile(self, obj, src, ext, cc_args, extra_postargs, pp_opts): - """Compiles the source by spawning GCC and windres if needed.""" - if ext == '.rc' or ext == '.res': - # gcc needs '.res' and '.rc' compiled to object files !!! - try: - self.spawn(["windres", "-i", src, "-o", obj]) - except DistutilsExecError as msg: - raise CompileError(msg) - else: # for other files use the C-compiler - try: - self.spawn(self.compiler_so + cc_args + [src, '-o', obj] + - extra_postargs) - except DistutilsExecError as msg: - raise CompileError(msg) - - def link(self, target_desc, objects, output_filename, output_dir=None, - libraries=None, library_dirs=None, runtime_library_dirs=None, - export_symbols=None, debug=0, extra_preargs=None, - extra_postargs=None, build_temp=None, target_lang=None): - """Link the objects.""" - # use separate copies, so we can modify the lists - extra_preargs = copy.copy(extra_preargs or []) - libraries = copy.copy(libraries or []) - objects = copy.copy(objects or []) - - # Additional libraries - libraries.extend(self.dll_libraries) - - # handle export symbols by creating a def-file - # with executables this only works with gcc/ld as linker - if ((export_symbols is not None) and - (target_desc != self.EXECUTABLE or self.linker_dll == "gcc")): - # (The linker doesn't do anything if output is up-to-date. - # So it would probably better to check if we really need this, - # but for this we had to insert some unchanged parts of - # UnixCCompiler, and this is not what we want.) - - # we want to put some files in the same directory as the - # object files are, build_temp doesn't help much - # where are the object files - temp_dir = os.path.dirname(objects[0]) - # name of dll to give the helper files the same base name - (dll_name, dll_extension) = os.path.splitext( - os.path.basename(output_filename)) - - # generate the filenames for these files - def_file = os.path.join(temp_dir, dll_name + ".def") - lib_file = os.path.join(temp_dir, 'lib' + dll_name + ".a") - - # Generate .def file - contents = [ - "LIBRARY %s" % os.path.basename(output_filename), - "EXPORTS"] - for sym in export_symbols: - contents.append(sym) - self.execute(write_file, (def_file, contents), - "writing %s" % def_file) - - # next add options for def-file and to creating import libraries - - # dllwrap uses different options than gcc/ld - if self.linker_dll == "dllwrap": - extra_preargs.extend(["--output-lib", lib_file]) - # for dllwrap we have to use a special option - extra_preargs.extend(["--def", def_file]) - # we use gcc/ld here and can be sure ld is >= 2.9.10 - else: - # doesn't work: bfd_close build\...\libfoo.a: Invalid operation - #extra_preargs.extend(["-Wl,--out-implib,%s" % lib_file]) - # for gcc/ld the def-file is specified as any object files - objects.append(def_file) - - #end: if ((export_symbols is not None) and - # (target_desc != self.EXECUTABLE or self.linker_dll == "gcc")): - - # who wants symbols and a many times larger output file - # should explicitly switch the debug mode on - # otherwise we let dllwrap/ld strip the output file - # (On my machine: 10KB < stripped_file < ??100KB - # unstripped_file = stripped_file + XXX KB - # ( XXX=254 for a typical python extension)) - if not debug: - extra_preargs.append("-s") - - UnixCCompiler.link(self, target_desc, objects, output_filename, - output_dir, libraries, library_dirs, - runtime_library_dirs, - None, # export_symbols, we do this in our def-file - debug, extra_preargs, extra_postargs, build_temp, - target_lang) - - # -- Miscellaneous methods ----------------------------------------- - - def object_filenames(self, source_filenames, strip_dir=0, output_dir=''): - """Adds supports for rc and res files.""" - if output_dir is None: - output_dir = '' - obj_names = [] - for src_name in source_filenames: - # use normcase to make sure '.rc' is really '.rc' and not '.RC' - base, ext = os.path.splitext(os.path.normcase(src_name)) - if ext not in (self.src_extensions + ['.rc','.res']): - raise UnknownFileError("unknown file type '%s' (from '%s')" % \ - (ext, src_name)) - if strip_dir: - base = os.path.basename (base) - if ext in ('.res', '.rc'): - # these need to be compiled to object files - obj_names.append (os.path.join(output_dir, - base + ext + self.obj_extension)) - else: - obj_names.append (os.path.join(output_dir, - base + self.obj_extension)) - return obj_names - -# the same as cygwin plus some additional parameters -class Mingw32CCompiler(CygwinCCompiler): - """ Handles the Mingw32 port of the GNU C compiler to Windows. - """ - compiler_type = 'mingw32' - - def __init__(self, verbose=0, dry_run=0, force=0): - - CygwinCCompiler.__init__ (self, verbose, dry_run, force) - - # ld_version >= "2.13" support -shared so use it instead of - # -mdll -static - if self.ld_version >= "2.13": - shared_option = "-shared" - else: - shared_option = "-mdll -static" - - # A real mingw32 doesn't need to specify a different entry point, - # but cygwin 2.91.57 in no-cygwin-mode needs it. - if self.gcc_version <= "2.91.57": - entry_point = '--entry _DllMain@12' - else: - entry_point = '' - - if is_cygwingcc(): - raise CCompilerError( - 'Cygwin gcc cannot be used with --compiler=mingw32') - - self.set_executables(compiler='gcc -O -Wall', - compiler_so='gcc -mdll -O -Wall', - compiler_cxx='g++ -O -Wall', - linker_exe='gcc', - linker_so='%s %s %s' - % (self.linker_dll, shared_option, - entry_point)) - # Maybe we should also append -mthreads, but then the finished - # dlls need another dll (mingwm10.dll see Mingw32 docs) - # (-mthreads: Support thread-safe exception handling on `Mingw32') - - # no additional libraries needed - self.dll_libraries=[] - - # Include the appropriate MSVC runtime library if Python was built - # with MSVC 7.0 or later. - self.dll_libraries = get_msvcr() - -# Because these compilers aren't configured in Python's pyconfig.h file by -# default, we should at least warn the user if he is using an unmodified -# version. - -CONFIG_H_OK = "ok" -CONFIG_H_NOTOK = "not ok" -CONFIG_H_UNCERTAIN = "uncertain" - -def check_config_h(): - """Check if the current Python installation appears amenable to building - extensions with GCC. - - Returns a tuple (status, details), where 'status' is one of the following - constants: - - - CONFIG_H_OK: all is well, go ahead and compile - - CONFIG_H_NOTOK: doesn't look good - - CONFIG_H_UNCERTAIN: not sure -- unable to read pyconfig.h - - 'details' is a human-readable string explaining the situation. - - Note there are two ways to conclude "OK": either 'sys.version' contains - the string "GCC" (implying that this Python was built with GCC), or the - installed "pyconfig.h" contains the string "__GNUC__". - """ - - # XXX since this function also checks sys.version, it's not strictly a - # "pyconfig.h" check -- should probably be renamed... - - from distutils import sysconfig - - # if sys.version contains GCC then python was compiled with GCC, and the - # pyconfig.h file should be OK - if "GCC" in sys.version: - return CONFIG_H_OK, "sys.version mentions 'GCC'" - - # let's see if __GNUC__ is mentioned in python.h - fn = sysconfig.get_config_h_filename() - try: - config_h = open(fn) - try: - if "__GNUC__" in config_h.read(): - return CONFIG_H_OK, "'%s' mentions '__GNUC__'" % fn - else: - return CONFIG_H_NOTOK, "'%s' does not mention '__GNUC__'" % fn - finally: - config_h.close() - except OSError as exc: - return (CONFIG_H_UNCERTAIN, - "couldn't read '%s': %s" % (fn, exc.strerror)) - -RE_VERSION = re.compile(br'(\d+\.\d+(\.\d+)*)') - -def _find_exe_version(cmd): - """Find the version of an executable by running `cmd` in the shell. - - If the command is not found, or the output does not match - `RE_VERSION`, returns None. - """ - executable = cmd.split()[0] - if find_executable(executable) is None: - return None - out = Popen(cmd, shell=True, stdout=PIPE).stdout - try: - out_string = out.read() - finally: - out.close() - result = RE_VERSION.search(out_string) - if result is None: - return None - # LooseVersion works with strings - # so we need to decode our bytes - return LooseVersion(result.group(1).decode()) - -def get_versions(): - """ Try to find out the versions of gcc, ld and dllwrap. - - If not possible it returns None for it. - """ - commands = ['gcc -dumpversion', 'ld -v', 'dllwrap --version'] - return tuple([_find_exe_version(cmd) for cmd in commands]) - -def is_cygwingcc(): - '''Try to determine if the gcc that would be used is from cygwin.''' - out_string = check_output(['gcc', '-dumpmachine']) - return out_string.strip().endswith(b'cygwin') diff --git a/Lib/distutils/debug.py b/Lib/distutils/debug.py deleted file mode 100644 index daf1660f0d8..00000000000 --- a/Lib/distutils/debug.py +++ /dev/null @@ -1,5 +0,0 @@ -import os - -# If DISTUTILS_DEBUG is anything other than the empty string, we run in -# debug mode. -DEBUG = os.environ.get('DISTUTILS_DEBUG') diff --git a/Lib/distutils/dep_util.py b/Lib/distutils/dep_util.py deleted file mode 100644 index d74f5e4e92f..00000000000 --- a/Lib/distutils/dep_util.py +++ /dev/null @@ -1,92 +0,0 @@ -"""distutils.dep_util - -Utility functions for simple, timestamp-based dependency of files -and groups of files; also, function based entirely on such -timestamp dependency analysis.""" - -import os -from distutils.errors import DistutilsFileError - - -def newer (source, target): - """Return true if 'source' exists and is more recently modified than - 'target', or if 'source' exists and 'target' doesn't. Return false if - both exist and 'target' is the same age or younger than 'source'. - Raise DistutilsFileError if 'source' does not exist. - """ - if not os.path.exists(source): - raise DistutilsFileError("file '%s' does not exist" % - os.path.abspath(source)) - if not os.path.exists(target): - return 1 - - from stat import ST_MTIME - mtime1 = os.stat(source)[ST_MTIME] - mtime2 = os.stat(target)[ST_MTIME] - - return mtime1 > mtime2 - -# newer () - - -def newer_pairwise (sources, targets): - """Walk two filename lists in parallel, testing if each source is newer - than its corresponding target. Return a pair of lists (sources, - targets) where source is newer than target, according to the semantics - of 'newer()'. - """ - if len(sources) != len(targets): - raise ValueError("'sources' and 'targets' must be same length") - - # build a pair of lists (sources, targets) where source is newer - n_sources = [] - n_targets = [] - for i in range(len(sources)): - if newer(sources[i], targets[i]): - n_sources.append(sources[i]) - n_targets.append(targets[i]) - - return (n_sources, n_targets) - -# newer_pairwise () - - -def newer_group (sources, target, missing='error'): - """Return true if 'target' is out-of-date with respect to any file - listed in 'sources'. In other words, if 'target' exists and is newer - than every file in 'sources', return false; otherwise return true. - 'missing' controls what we do when a source file is missing; the - default ("error") is to blow up with an OSError from inside 'stat()'; - if it is "ignore", we silently drop any missing source files; if it is - "newer", any missing source files make us assume that 'target' is - out-of-date (this is handy in "dry-run" mode: it'll make you pretend to - carry out commands that wouldn't work because inputs are missing, but - that doesn't matter because you're not actually going to run the - commands). - """ - # If the target doesn't even exist, then it's definitely out-of-date. - if not os.path.exists(target): - return 1 - - # Otherwise we have to find out the hard way: if *any* source file - # is more recent than 'target', then 'target' is out-of-date and - # we can immediately return true. If we fall through to the end - # of the loop, then 'target' is up-to-date and we return false. - from stat import ST_MTIME - target_mtime = os.stat(target)[ST_MTIME] - for source in sources: - if not os.path.exists(source): - if missing == 'error': # blow up when we stat() the file - pass - elif missing == 'ignore': # missing source dropped from - continue # target's dependency list - elif missing == 'newer': # missing source means target is - return 1 # out-of-date - - source_mtime = os.stat(source)[ST_MTIME] - if source_mtime > target_mtime: - return 1 - else: - return 0 - -# newer_group () diff --git a/Lib/distutils/dir_util.py b/Lib/distutils/dir_util.py deleted file mode 100644 index df4d751c942..00000000000 --- a/Lib/distutils/dir_util.py +++ /dev/null @@ -1,223 +0,0 @@ -"""distutils.dir_util - -Utility functions for manipulating directories and directory trees.""" - -import os -import errno -from distutils.errors import DistutilsFileError, DistutilsInternalError -from distutils import log - -# cache for by mkpath() -- in addition to cheapening redundant calls, -# eliminates redundant "creating /foo/bar/baz" messages in dry-run mode -_path_created = {} - -# I don't use os.makedirs because a) it's new to Python 1.5.2, and -# b) it blows up if the directory already exists (I want to silently -# succeed in that case). -def mkpath(name, mode=0o777, verbose=1, dry_run=0): - """Create a directory and any missing ancestor directories. - - If the directory already exists (or if 'name' is the empty string, which - means the current directory, which of course exists), then do nothing. - Raise DistutilsFileError if unable to create some directory along the way - (eg. some sub-path exists, but is a file rather than a directory). - If 'verbose' is true, print a one-line summary of each mkdir to stdout. - Return the list of directories actually created. - """ - - global _path_created - - # Detect a common bug -- name is None - if not isinstance(name, str): - raise DistutilsInternalError( - "mkpath: 'name' must be a string (got %r)" % (name,)) - - # XXX what's the better way to handle verbosity? print as we create - # each directory in the path (the current behaviour), or only announce - # the creation of the whole path? (quite easy to do the latter since - # we're not using a recursive algorithm) - - name = os.path.normpath(name) - created_dirs = [] - if os.path.isdir(name) or name == '': - return created_dirs - if _path_created.get(os.path.abspath(name)): - return created_dirs - - (head, tail) = os.path.split(name) - tails = [tail] # stack of lone dirs to create - - while head and tail and not os.path.isdir(head): - (head, tail) = os.path.split(head) - tails.insert(0, tail) # push next higher dir onto stack - - # now 'head' contains the deepest directory that already exists - # (that is, the child of 'head' in 'name' is the highest directory - # that does *not* exist) - for d in tails: - #print "head = %s, d = %s: " % (head, d), - head = os.path.join(head, d) - abs_head = os.path.abspath(head) - - if _path_created.get(abs_head): - continue - - if verbose >= 1: - log.info("creating %s", head) - - if not dry_run: - try: - os.mkdir(head, mode) - except OSError as exc: - if not (exc.errno == errno.EEXIST and os.path.isdir(head)): - raise DistutilsFileError( - "could not create '%s': %s" % (head, exc.args[-1])) - created_dirs.append(head) - - _path_created[abs_head] = 1 - return created_dirs - -def create_tree(base_dir, files, mode=0o777, verbose=1, dry_run=0): - """Create all the empty directories under 'base_dir' needed to put 'files' - there. - - 'base_dir' is just the name of a directory which doesn't necessarily - exist yet; 'files' is a list of filenames to be interpreted relative to - 'base_dir'. 'base_dir' + the directory portion of every file in 'files' - will be created if it doesn't already exist. 'mode', 'verbose' and - 'dry_run' flags are as for 'mkpath()'. - """ - # First get the list of directories to create - need_dir = set() - for file in files: - need_dir.add(os.path.join(base_dir, os.path.dirname(file))) - - # Now create them - for dir in sorted(need_dir): - mkpath(dir, mode, verbose=verbose, dry_run=dry_run) - -import sysconfig -_multiarch = None - -def copy_tree(src, dst, preserve_mode=1, preserve_times=1, - preserve_symlinks=0, update=0, verbose=1, dry_run=0): - """Copy an entire directory tree 'src' to a new location 'dst'. - - Both 'src' and 'dst' must be directory names. If 'src' is not a - directory, raise DistutilsFileError. If 'dst' does not exist, it is - created with 'mkpath()'. The end result of the copy is that every - file in 'src' is copied to 'dst', and directories under 'src' are - recursively copied to 'dst'. Return the list of files that were - copied or might have been copied, using their output name. The - return value is unaffected by 'update' or 'dry_run': it is simply - the list of all files under 'src', with the names changed to be - under 'dst'. - - 'preserve_mode' and 'preserve_times' are the same as for - 'copy_file'; note that they only apply to regular files, not to - directories. If 'preserve_symlinks' is true, symlinks will be - copied as symlinks (on platforms that support them!); otherwise - (the default), the destination of the symlink will be copied. - 'update' and 'verbose' are the same as for 'copy_file'. - """ - from distutils.file_util import copy_file - - if not dry_run and not os.path.isdir(src): - raise DistutilsFileError( - "cannot copy tree '%s': not a directory" % src) - try: - names = os.listdir(src) - except OSError as e: - if dry_run: - names = [] - else: - raise DistutilsFileError( - "error listing files in '%s': %s" % (src, e.strerror)) - - ext_suffix = sysconfig.get_config_var ('EXT_SUFFIX') - _multiarch = sysconfig.get_config_var ('MULTIARCH') - if ext_suffix.endswith(_multiarch + ext_suffix[-3:]): - new_suffix = None - else: - new_suffix = "%s-%s%s" % (ext_suffix[:-3], _multiarch, ext_suffix[-3:]) - - if not dry_run: - mkpath(dst, verbose=verbose) - - outputs = [] - - for n in names: - src_name = os.path.join(src, n) - dst_name = os.path.join(dst, n) - if new_suffix and _multiarch and n.endswith(ext_suffix) and not n.endswith(new_suffix): - dst_name = os.path.join(dst, n.replace(ext_suffix, new_suffix)) - log.info("renaming extension %s -> %s", n, n.replace(ext_suffix, new_suffix)) - - if n.startswith('.nfs'): - # skip NFS rename files - continue - - if preserve_symlinks and os.path.islink(src_name): - link_dest = os.readlink(src_name) - if verbose >= 1: - log.info("linking %s -> %s", dst_name, link_dest) - if not dry_run: - os.symlink(link_dest, dst_name) - outputs.append(dst_name) - - elif os.path.isdir(src_name): - outputs.extend( - copy_tree(src_name, dst_name, preserve_mode, - preserve_times, preserve_symlinks, update, - verbose=verbose, dry_run=dry_run)) - else: - copy_file(src_name, dst_name, preserve_mode, - preserve_times, update, verbose=verbose, - dry_run=dry_run) - outputs.append(dst_name) - - return outputs - -def _build_cmdtuple(path, cmdtuples): - """Helper for remove_tree().""" - for f in os.listdir(path): - real_f = os.path.join(path,f) - if os.path.isdir(real_f) and not os.path.islink(real_f): - _build_cmdtuple(real_f, cmdtuples) - else: - cmdtuples.append((os.remove, real_f)) - cmdtuples.append((os.rmdir, path)) - -def remove_tree(directory, verbose=1, dry_run=0): - """Recursively remove an entire directory tree. - - Any errors are ignored (apart from being reported to stdout if 'verbose' - is true). - """ - global _path_created - - if verbose >= 1: - log.info("removing '%s' (and everything under it)", directory) - if dry_run: - return - cmdtuples = [] - _build_cmdtuple(directory, cmdtuples) - for cmd in cmdtuples: - try: - cmd[0](cmd[1]) - # remove dir from cache if it's already there - abspath = os.path.abspath(cmd[1]) - if abspath in _path_created: - del _path_created[abspath] - except OSError as exc: - log.warn("error removing %s: %s", directory, exc) - -def ensure_relative(path): - """Take the full path 'path', and make it a relative path. - - This is useful to make 'path' the second argument to os.path.join(). - """ - drive, path = os.path.splitdrive(path) - if path[0:1] == os.sep: - path = drive + path[1:] - return path diff --git a/Lib/distutils/dist.py b/Lib/distutils/dist.py deleted file mode 100644 index 62a24516cfa..00000000000 --- a/Lib/distutils/dist.py +++ /dev/null @@ -1,1236 +0,0 @@ -"""distutils.dist - -Provides the Distribution class, which represents the module distribution -being built/installed/distributed. -""" - -import sys -import os -import re -from email import message_from_file - -try: - import warnings -except ImportError: - warnings = None - -from distutils.errors import * -from distutils.fancy_getopt import FancyGetopt, translate_longopt -from distutils.util import check_environ, strtobool, rfc822_escape -from distutils import log -from distutils.debug import DEBUG - -# Regex to define acceptable Distutils command names. This is not *quite* -# the same as a Python NAME -- I don't allow leading underscores. The fact -# that they're very similar is no coincidence; the default naming scheme is -# to look for a Python module named after the command. -command_re = re.compile(r'^[a-zA-Z]([a-zA-Z0-9_]*)$') - - -class Distribution: - """The core of the Distutils. Most of the work hiding behind 'setup' - is really done within a Distribution instance, which farms the work out - to the Distutils commands specified on the command line. - - Setup scripts will almost never instantiate Distribution directly, - unless the 'setup()' function is totally inadequate to their needs. - However, it is conceivable that a setup script might wish to subclass - Distribution for some specialized purpose, and then pass the subclass - to 'setup()' as the 'distclass' keyword argument. If so, it is - necessary to respect the expectations that 'setup' has of Distribution. - See the code for 'setup()', in core.py, for details. - """ - - # 'global_options' describes the command-line options that may be - # supplied to the setup script prior to any actual commands. - # Eg. "./setup.py -n" or "./setup.py --quiet" both take advantage of - # these global options. This list should be kept to a bare minimum, - # since every global option is also valid as a command option -- and we - # don't want to pollute the commands with too many options that they - # have minimal control over. - # The fourth entry for verbose means that it can be repeated. - global_options = [ - ('verbose', 'v', "run verbosely (default)", 1), - ('quiet', 'q', "run quietly (turns verbosity off)"), - ('dry-run', 'n', "don't actually do anything"), - ('help', 'h', "show detailed help message"), - ('no-user-cfg', None, - 'ignore pydistutils.cfg in your home directory'), - ] - - # 'common_usage' is a short (2-3 line) string describing the common - # usage of the setup script. - common_usage = """\ -Common commands: (see '--help-commands' for more) - - setup.py build will build the package underneath 'build/' - setup.py install will install the package -""" - - # options that are not propagated to the commands - display_options = [ - ('help-commands', None, - "list all available commands"), - ('name', None, - "print package name"), - ('version', 'V', - "print package version"), - ('fullname', None, - "print -"), - ('author', None, - "print the author's name"), - ('author-email', None, - "print the author's email address"), - ('maintainer', None, - "print the maintainer's name"), - ('maintainer-email', None, - "print the maintainer's email address"), - ('contact', None, - "print the maintainer's name if known, else the author's"), - ('contact-email', None, - "print the maintainer's email address if known, else the author's"), - ('url', None, - "print the URL for this package"), - ('license', None, - "print the license of the package"), - ('licence', None, - "alias for --license"), - ('description', None, - "print the package description"), - ('long-description', None, - "print the long package description"), - ('platforms', None, - "print the list of platforms"), - ('classifiers', None, - "print the list of classifiers"), - ('keywords', None, - "print the list of keywords"), - ('provides', None, - "print the list of packages/modules provided"), - ('requires', None, - "print the list of packages/modules required"), - ('obsoletes', None, - "print the list of packages/modules made obsolete") - ] - display_option_names = [translate_longopt(x[0]) for x in display_options] - - # negative options are options that exclude other options - negative_opt = {'quiet': 'verbose'} - - # -- Creation/initialization methods ------------------------------- - - def __init__(self, attrs=None): - """Construct a new Distribution instance: initialize all the - attributes of a Distribution, and then use 'attrs' (a dictionary - mapping attribute names to values) to assign some of those - attributes their "real" values. (Any attributes not mentioned in - 'attrs' will be assigned to some null value: 0, None, an empty list - or dictionary, etc.) Most importantly, initialize the - 'command_obj' attribute to the empty dictionary; this will be - filled in with real command objects by 'parse_command_line()'. - """ - - # Default values for our command-line options - self.verbose = 1 - self.dry_run = 0 - self.help = 0 - for attr in self.display_option_names: - setattr(self, attr, 0) - - # Store the distribution meta-data (name, version, author, and so - # forth) in a separate object -- we're getting to have enough - # information here (and enough command-line options) that it's - # worth it. Also delegate 'get_XXX()' methods to the 'metadata' - # object in a sneaky and underhanded (but efficient!) way. - self.metadata = DistributionMetadata() - for basename in self.metadata._METHOD_BASENAMES: - method_name = "get_" + basename - setattr(self, method_name, getattr(self.metadata, method_name)) - - # 'cmdclass' maps command names to class objects, so we - # can 1) quickly figure out which class to instantiate when - # we need to create a new command object, and 2) have a way - # for the setup script to override command classes - self.cmdclass = {} - - # 'command_packages' is a list of packages in which commands - # are searched for. The factory for command 'foo' is expected - # to be named 'foo' in the module 'foo' in one of the packages - # named here. This list is searched from the left; an error - # is raised if no named package provides the command being - # searched for. (Always access using get_command_packages().) - self.command_packages = None - - # 'script_name' and 'script_args' are usually set to sys.argv[0] - # and sys.argv[1:], but they can be overridden when the caller is - # not necessarily a setup script run from the command-line. - self.script_name = None - self.script_args = None - - # 'command_options' is where we store command options between - # parsing them (from config files, the command-line, etc.) and when - # they are actually needed -- ie. when the command in question is - # instantiated. It is a dictionary of dictionaries of 2-tuples: - # command_options = { command_name : { option : (source, value) } } - self.command_options = {} - - # 'dist_files' is the list of (command, pyversion, file) that - # have been created by any dist commands run so far. This is - # filled regardless of whether the run is dry or not. pyversion - # gives sysconfig.get_python_version() if the dist file is - # specific to a Python version, 'any' if it is good for all - # Python versions on the target platform, and '' for a source - # file. pyversion should not be used to specify minimum or - # maximum required Python versions; use the metainfo for that - # instead. - self.dist_files = [] - - # These options are really the business of various commands, rather - # than of the Distribution itself. We provide aliases for them in - # Distribution as a convenience to the developer. - self.packages = None - self.package_data = {} - self.package_dir = None - self.py_modules = None - self.libraries = None - self.headers = None - self.ext_modules = None - self.ext_package = None - self.include_dirs = None - self.extra_path = None - self.scripts = None - self.data_files = None - self.password = '' - - # And now initialize bookkeeping stuff that can't be supplied by - # the caller at all. 'command_obj' maps command names to - # Command instances -- that's how we enforce that every command - # class is a singleton. - self.command_obj = {} - - # 'have_run' maps command names to boolean values; it keeps track - # of whether we have actually run a particular command, to make it - # cheap to "run" a command whenever we think we might need to -- if - # it's already been done, no need for expensive filesystem - # operations, we just check the 'have_run' dictionary and carry on. - # It's only safe to query 'have_run' for a command class that has - # been instantiated -- a false value will be inserted when the - # command object is created, and replaced with a true value when - # the command is successfully run. Thus it's probably best to use - # '.get()' rather than a straight lookup. - self.have_run = {} - - # Now we'll use the attrs dictionary (ultimately, keyword args from - # the setup script) to possibly override any or all of these - # distribution options. - - if attrs: - # Pull out the set of command options and work on them - # specifically. Note that this order guarantees that aliased - # command options will override any supplied redundantly - # through the general options dictionary. - options = attrs.get('options') - if options is not None: - del attrs['options'] - for (command, cmd_options) in options.items(): - opt_dict = self.get_option_dict(command) - for (opt, val) in cmd_options.items(): - opt_dict[opt] = ("setup script", val) - - if 'licence' in attrs: - attrs['license'] = attrs['licence'] - del attrs['licence'] - msg = "'licence' distribution option is deprecated; use 'license'" - if warnings is not None: - warnings.warn(msg) - else: - sys.stderr.write(msg + "\n") - - # Now work on the rest of the attributes. Any attribute that's - # not already defined is invalid! - for (key, val) in attrs.items(): - if hasattr(self.metadata, "set_" + key): - getattr(self.metadata, "set_" + key)(val) - elif hasattr(self.metadata, key): - setattr(self.metadata, key, val) - elif hasattr(self, key): - setattr(self, key, val) - else: - msg = "Unknown distribution option: %s" % repr(key) - if warnings is not None: - warnings.warn(msg) - else: - sys.stderr.write(msg + "\n") - - # no-user-cfg is handled before other command line args - # because other args override the config files, and this - # one is needed before we can load the config files. - # If attrs['script_args'] wasn't passed, assume false. - # - # This also make sure we just look at the global options - self.want_user_cfg = True - - if self.script_args is not None: - for arg in self.script_args: - if not arg.startswith('-'): - break - if arg == '--no-user-cfg': - self.want_user_cfg = False - break - - self.finalize_options() - - def get_option_dict(self, command): - """Get the option dictionary for a given command. If that - command's option dictionary hasn't been created yet, then create it - and return the new dictionary; otherwise, return the existing - option dictionary. - """ - dict = self.command_options.get(command) - if dict is None: - dict = self.command_options[command] = {} - return dict - - def dump_option_dicts(self, header=None, commands=None, indent=""): - from pprint import pformat - - if commands is None: # dump all command option dicts - commands = sorted(self.command_options.keys()) - - if header is not None: - self.announce(indent + header) - indent = indent + " " - - if not commands: - self.announce(indent + "no commands known yet") - return - - for cmd_name in commands: - opt_dict = self.command_options.get(cmd_name) - if opt_dict is None: - self.announce(indent + - "no option dict for '%s' command" % cmd_name) - else: - self.announce(indent + - "option dict for '%s' command:" % cmd_name) - out = pformat(opt_dict) - for line in out.split('\n'): - self.announce(indent + " " + line) - - # -- Config file finding/parsing methods --------------------------- - - def find_config_files(self): - """Find as many configuration files as should be processed for this - platform, and return a list of filenames in the order in which they - should be parsed. The filenames returned are guaranteed to exist - (modulo nasty race conditions). - - There are three possible config files: distutils.cfg in the - Distutils installation directory (ie. where the top-level - Distutils __inst__.py file lives), a file in the user's home - directory named .pydistutils.cfg on Unix and pydistutils.cfg - on Windows/Mac; and setup.cfg in the current directory. - - The file in the user's home directory can be disabled with the - --no-user-cfg option. - """ - files = [] - check_environ() - - # Where to look for the system-wide Distutils config file - sys_dir = os.path.dirname(sys.modules['distutils'].__file__) - - # Look for the system config file - sys_file = os.path.join(sys_dir, "distutils.cfg") - if os.path.isfile(sys_file): - files.append(sys_file) - - # What to call the per-user config file - if os.name == 'posix': - user_filename = ".pydistutils.cfg" - else: - user_filename = "pydistutils.cfg" - - # And look for the user config file - if self.want_user_cfg: - user_file = os.path.join(os.path.expanduser('~'), user_filename) - if os.path.isfile(user_file): - files.append(user_file) - - # All platforms support local setup.cfg - local_file = "setup.cfg" - if os.path.isfile(local_file): - files.append(local_file) - - if DEBUG: - self.announce("using config files: %s" % ', '.join(files)) - - return files - - def parse_config_files(self, filenames=None): - from configparser import ConfigParser - - # Ignore install directory options if we have a venv - if sys.prefix != sys.base_prefix: - ignore_options = [ - 'install-base', 'install-platbase', 'install-lib', - 'install-platlib', 'install-purelib', 'install-headers', - 'install-scripts', 'install-data', 'prefix', 'exec-prefix', - 'home', 'user', 'root'] - else: - ignore_options = [] - - ignore_options = frozenset(ignore_options) - - if filenames is None: - filenames = self.find_config_files() - - if DEBUG: - self.announce("Distribution.parse_config_files():") - - parser = ConfigParser() - for filename in filenames: - if DEBUG: - self.announce(" reading %s" % filename) - parser.read(filename) - for section in parser.sections(): - options = parser.options(section) - opt_dict = self.get_option_dict(section) - - for opt in options: - if opt != '__name__' and opt not in ignore_options: - val = parser.get(section,opt) - opt = opt.replace('-', '_') - opt_dict[opt] = (filename, val) - - # Make the ConfigParser forget everything (so we retain - # the original filenames that options come from) - parser.__init__() - - # If there was a "global" section in the config file, use it - # to set Distribution options. - - if 'global' in self.command_options: - for (opt, (src, val)) in self.command_options['global'].items(): - alias = self.negative_opt.get(opt) - try: - if alias: - setattr(self, alias, not strtobool(val)) - elif opt in ('verbose', 'dry_run'): # ugh! - setattr(self, opt, strtobool(val)) - else: - setattr(self, opt, val) - except ValueError as msg: - raise DistutilsOptionError(msg) - - # -- Command-line parsing methods ---------------------------------- - - def parse_command_line(self): - """Parse the setup script's command line, taken from the - 'script_args' instance attribute (which defaults to 'sys.argv[1:]' - -- see 'setup()' in core.py). This list is first processed for - "global options" -- options that set attributes of the Distribution - instance. Then, it is alternately scanned for Distutils commands - and options for that command. Each new command terminates the - options for the previous command. The allowed options for a - command are determined by the 'user_options' attribute of the - command class -- thus, we have to be able to load command classes - in order to parse the command line. Any error in that 'options' - attribute raises DistutilsGetoptError; any error on the - command-line raises DistutilsArgError. If no Distutils commands - were found on the command line, raises DistutilsArgError. Return - true if command-line was successfully parsed and we should carry - on with executing commands; false if no errors but we shouldn't - execute commands (currently, this only happens if user asks for - help). - """ - # - # We now have enough information to show the Macintosh dialog - # that allows the user to interactively specify the "command line". - # - toplevel_options = self._get_toplevel_options() - - # We have to parse the command line a bit at a time -- global - # options, then the first command, then its options, and so on -- - # because each command will be handled by a different class, and - # the options that are valid for a particular class aren't known - # until we have loaded the command class, which doesn't happen - # until we know what the command is. - - self.commands = [] - parser = FancyGetopt(toplevel_options + self.display_options) - parser.set_negative_aliases(self.negative_opt) - parser.set_aliases({'licence': 'license'}) - args = parser.getopt(args=self.script_args, object=self) - option_order = parser.get_option_order() - log.set_verbosity(self.verbose) - - # for display options we return immediately - if self.handle_display_options(option_order): - return - while args: - args = self._parse_command_opts(parser, args) - if args is None: # user asked for help (and got it) - return - - # Handle the cases of --help as a "global" option, ie. - # "setup.py --help" and "setup.py --help command ...". For the - # former, we show global options (--verbose, --dry-run, etc.) - # and display-only options (--name, --version, etc.); for the - # latter, we omit the display-only options and show help for - # each command listed on the command line. - if self.help: - self._show_help(parser, - display_options=len(self.commands) == 0, - commands=self.commands) - return - - # Oops, no commands found -- an end-user error - if not self.commands: - raise DistutilsArgError("no commands supplied") - - # All is well: return true - return True - - def _get_toplevel_options(self): - """Return the non-display options recognized at the top level. - - This includes options that are recognized *only* at the top - level as well as options recognized for commands. - """ - return self.global_options + [ - ("command-packages=", None, - "list of packages that provide distutils commands"), - ] - - def _parse_command_opts(self, parser, args): - """Parse the command-line options for a single command. - 'parser' must be a FancyGetopt instance; 'args' must be the list - of arguments, starting with the current command (whose options - we are about to parse). Returns a new version of 'args' with - the next command at the front of the list; will be the empty - list if there are no more commands on the command line. Returns - None if the user asked for help on this command. - """ - # late import because of mutual dependence between these modules - from distutils.cmd import Command - - # Pull the current command from the head of the command line - command = args[0] - if not command_re.match(command): - raise SystemExit("invalid command name '%s'" % command) - self.commands.append(command) - - # Dig up the command class that implements this command, so we - # 1) know that it's a valid command, and 2) know which options - # it takes. - try: - cmd_class = self.get_command_class(command) - except DistutilsModuleError as msg: - raise DistutilsArgError(msg) - - # Require that the command class be derived from Command -- want - # to be sure that the basic "command" interface is implemented. - if not issubclass(cmd_class, Command): - raise DistutilsClassError( - "command class %s must subclass Command" % cmd_class) - - # Also make sure that the command object provides a list of its - # known options. - if not (hasattr(cmd_class, 'user_options') and - isinstance(cmd_class.user_options, list)): - msg = ("command class %s must provide " - "'user_options' attribute (a list of tuples)") - raise DistutilsClassError(msg % cmd_class) - - # If the command class has a list of negative alias options, - # merge it in with the global negative aliases. - negative_opt = self.negative_opt - if hasattr(cmd_class, 'negative_opt'): - negative_opt = negative_opt.copy() - negative_opt.update(cmd_class.negative_opt) - - # Check for help_options in command class. They have a different - # format (tuple of four) so we need to preprocess them here. - if (hasattr(cmd_class, 'help_options') and - isinstance(cmd_class.help_options, list)): - help_options = fix_help_options(cmd_class.help_options) - else: - help_options = [] - - # All commands support the global options too, just by adding - # in 'global_options'. - parser.set_option_table(self.global_options + - cmd_class.user_options + - help_options) - parser.set_negative_aliases(negative_opt) - (args, opts) = parser.getopt(args[1:]) - if hasattr(opts, 'help') and opts.help: - self._show_help(parser, display_options=0, commands=[cmd_class]) - return - - if (hasattr(cmd_class, 'help_options') and - isinstance(cmd_class.help_options, list)): - help_option_found=0 - for (help_option, short, desc, func) in cmd_class.help_options: - if hasattr(opts, parser.get_attr_name(help_option)): - help_option_found=1 - if callable(func): - func() - else: - raise DistutilsClassError( - "invalid help function %r for help option '%s': " - "must be a callable object (function, etc.)" - % (func, help_option)) - - if help_option_found: - return - - # Put the options from the command-line into their official - # holding pen, the 'command_options' dictionary. - opt_dict = self.get_option_dict(command) - for (name, value) in vars(opts).items(): - opt_dict[name] = ("command line", value) - - return args - - def finalize_options(self): - """Set final values for all the options on the Distribution - instance, analogous to the .finalize_options() method of Command - objects. - """ - for attr in ('keywords', 'platforms'): - value = getattr(self.metadata, attr) - if value is None: - continue - if isinstance(value, str): - value = [elm.strip() for elm in value.split(',')] - setattr(self.metadata, attr, value) - - def _show_help(self, parser, global_options=1, display_options=1, - commands=[]): - """Show help for the setup script command-line in the form of - several lists of command-line options. 'parser' should be a - FancyGetopt instance; do not expect it to be returned in the - same state, as its option table will be reset to make it - generate the correct help text. - - If 'global_options' is true, lists the global options: - --verbose, --dry-run, etc. If 'display_options' is true, lists - the "display-only" options: --name, --version, etc. Finally, - lists per-command help for every command name or command class - in 'commands'. - """ - # late import because of mutual dependence between these modules - from distutils.core import gen_usage - from distutils.cmd import Command - - if global_options: - if display_options: - options = self._get_toplevel_options() - else: - options = self.global_options - parser.set_option_table(options) - parser.print_help(self.common_usage + "\nGlobal options:") - print('') - - if display_options: - parser.set_option_table(self.display_options) - parser.print_help( - "Information display options (just display " + - "information, ignore any commands)") - print('') - - for command in self.commands: - if isinstance(command, type) and issubclass(command, Command): - klass = command - else: - klass = self.get_command_class(command) - if (hasattr(klass, 'help_options') and - isinstance(klass.help_options, list)): - parser.set_option_table(klass.user_options + - fix_help_options(klass.help_options)) - else: - parser.set_option_table(klass.user_options) - parser.print_help("Options for '%s' command:" % klass.__name__) - print('') - - print(gen_usage(self.script_name)) - - def handle_display_options(self, option_order): - """If there were any non-global "display-only" options - (--help-commands or the metadata display options) on the command - line, display the requested info and return true; else return - false. - """ - from distutils.core import gen_usage - - # User just wants a list of commands -- we'll print it out and stop - # processing now (ie. if they ran "setup --help-commands foo bar", - # we ignore "foo bar"). - if self.help_commands: - self.print_commands() - print('') - print(gen_usage(self.script_name)) - return 1 - - # If user supplied any of the "display metadata" options, then - # display that metadata in the order in which the user supplied the - # metadata options. - any_display_options = 0 - is_display_option = {} - for option in self.display_options: - is_display_option[option[0]] = 1 - - for (opt, val) in option_order: - if val and is_display_option.get(opt): - opt = translate_longopt(opt) - value = getattr(self.metadata, "get_"+opt)() - if opt in ['keywords', 'platforms']: - print(','.join(value)) - elif opt in ('classifiers', 'provides', 'requires', - 'obsoletes'): - print('\n'.join(value)) - else: - print(value) - any_display_options = 1 - - return any_display_options - - def print_command_list(self, commands, header, max_length): - """Print a subset of the list of all commands -- used by - 'print_commands()'. - """ - print(header + ":") - - for cmd in commands: - klass = self.cmdclass.get(cmd) - if not klass: - klass = self.get_command_class(cmd) - try: - description = klass.description - except AttributeError: - description = "(no description available)" - - print(" %-*s %s" % (max_length, cmd, description)) - - def print_commands(self): - """Print out a help message listing all available commands with a - description of each. The list is divided into "standard commands" - (listed in distutils.command.__all__) and "extra commands" - (mentioned in self.cmdclass, but not a standard command). The - descriptions come from the command class attribute - 'description'. - """ - import distutils.command - std_commands = distutils.command.__all__ - is_std = {} - for cmd in std_commands: - is_std[cmd] = 1 - - extra_commands = [] - for cmd in self.cmdclass.keys(): - if not is_std.get(cmd): - extra_commands.append(cmd) - - max_length = 0 - for cmd in (std_commands + extra_commands): - if len(cmd) > max_length: - max_length = len(cmd) - - self.print_command_list(std_commands, - "Standard commands", - max_length) - if extra_commands: - print() - self.print_command_list(extra_commands, - "Extra commands", - max_length) - - def get_command_list(self): - """Get a list of (command, description) tuples. - The list is divided into "standard commands" (listed in - distutils.command.__all__) and "extra commands" (mentioned in - self.cmdclass, but not a standard command). The descriptions come - from the command class attribute 'description'. - """ - # Currently this is only used on Mac OS, for the Mac-only GUI - # Distutils interface (by Jack Jansen) - import distutils.command - std_commands = distutils.command.__all__ - is_std = {} - for cmd in std_commands: - is_std[cmd] = 1 - - extra_commands = [] - for cmd in self.cmdclass.keys(): - if not is_std.get(cmd): - extra_commands.append(cmd) - - rv = [] - for cmd in (std_commands + extra_commands): - klass = self.cmdclass.get(cmd) - if not klass: - klass = self.get_command_class(cmd) - try: - description = klass.description - except AttributeError: - description = "(no description available)" - rv.append((cmd, description)) - return rv - - # -- Command class/object methods ---------------------------------- - - def get_command_packages(self): - """Return a list of packages from which commands are loaded.""" - pkgs = self.command_packages - if not isinstance(pkgs, list): - if pkgs is None: - pkgs = '' - pkgs = [pkg.strip() for pkg in pkgs.split(',') if pkg != ''] - if "distutils.command" not in pkgs: - pkgs.insert(0, "distutils.command") - self.command_packages = pkgs - return pkgs - - def get_command_class(self, command): - """Return the class that implements the Distutils command named by - 'command'. First we check the 'cmdclass' dictionary; if the - command is mentioned there, we fetch the class object from the - dictionary and return it. Otherwise we load the command module - ("distutils.command." + command) and fetch the command class from - the module. The loaded class is also stored in 'cmdclass' - to speed future calls to 'get_command_class()'. - - Raises DistutilsModuleError if the expected module could not be - found, or if that module does not define the expected class. - """ - klass = self.cmdclass.get(command) - if klass: - return klass - - for pkgname in self.get_command_packages(): - module_name = "%s.%s" % (pkgname, command) - klass_name = command - - try: - __import__(module_name) - module = sys.modules[module_name] - except ImportError: - continue - - try: - klass = getattr(module, klass_name) - except AttributeError: - raise DistutilsModuleError( - "invalid command '%s' (no class '%s' in module '%s')" - % (command, klass_name, module_name)) - - self.cmdclass[command] = klass - return klass - - raise DistutilsModuleError("invalid command '%s'" % command) - - def get_command_obj(self, command, create=1): - """Return the command object for 'command'. Normally this object - is cached on a previous call to 'get_command_obj()'; if no command - object for 'command' is in the cache, then we either create and - return it (if 'create' is true) or return None. - """ - cmd_obj = self.command_obj.get(command) - if not cmd_obj and create: - if DEBUG: - self.announce("Distribution.get_command_obj(): " - "creating '%s' command object" % command) - - klass = self.get_command_class(command) - cmd_obj = self.command_obj[command] = klass(self) - self.have_run[command] = 0 - - # Set any options that were supplied in config files - # or on the command line. (NB. support for error - # reporting is lame here: any errors aren't reported - # until 'finalize_options()' is called, which means - # we won't report the source of the error.) - options = self.command_options.get(command) - if options: - self._set_command_options(cmd_obj, options) - - return cmd_obj - - def _set_command_options(self, command_obj, option_dict=None): - """Set the options for 'command_obj' from 'option_dict'. Basically - this means copying elements of a dictionary ('option_dict') to - attributes of an instance ('command'). - - 'command_obj' must be a Command instance. If 'option_dict' is not - supplied, uses the standard option dictionary for this command - (from 'self.command_options'). - """ - command_name = command_obj.get_command_name() - if option_dict is None: - option_dict = self.get_option_dict(command_name) - - if DEBUG: - self.announce(" setting options for '%s' command:" % command_name) - for (option, (source, value)) in option_dict.items(): - if DEBUG: - self.announce(" %s = %s (from %s)" % (option, value, - source)) - try: - bool_opts = [translate_longopt(o) - for o in command_obj.boolean_options] - except AttributeError: - bool_opts = [] - try: - neg_opt = command_obj.negative_opt - except AttributeError: - neg_opt = {} - - try: - is_string = isinstance(value, str) - if option in neg_opt and is_string: - setattr(command_obj, neg_opt[option], not strtobool(value)) - elif option in bool_opts and is_string: - setattr(command_obj, option, strtobool(value)) - elif hasattr(command_obj, option): - setattr(command_obj, option, value) - else: - raise DistutilsOptionError( - "error in %s: command '%s' has no such option '%s'" - % (source, command_name, option)) - except ValueError as msg: - raise DistutilsOptionError(msg) - - def reinitialize_command(self, command, reinit_subcommands=0): - """Reinitializes a command to the state it was in when first - returned by 'get_command_obj()': ie., initialized but not yet - finalized. This provides the opportunity to sneak option - values in programmatically, overriding or supplementing - user-supplied values from the config files and command line. - You'll have to re-finalize the command object (by calling - 'finalize_options()' or 'ensure_finalized()') before using it for - real. - - 'command' should be a command name (string) or command object. If - 'reinit_subcommands' is true, also reinitializes the command's - sub-commands, as declared by the 'sub_commands' class attribute (if - it has one). See the "install" command for an example. Only - reinitializes the sub-commands that actually matter, ie. those - whose test predicates return true. - - Returns the reinitialized command object. - """ - from distutils.cmd import Command - if not isinstance(command, Command): - command_name = command - command = self.get_command_obj(command_name) - else: - command_name = command.get_command_name() - - if not command.finalized: - return command - command.initialize_options() - command.finalized = 0 - self.have_run[command_name] = 0 - self._set_command_options(command) - - if reinit_subcommands: - for sub in command.get_sub_commands(): - self.reinitialize_command(sub, reinit_subcommands) - - return command - - # -- Methods that operate on the Distribution ---------------------- - - def announce(self, msg, level=log.INFO): - log.log(level, msg) - - def run_commands(self): - """Run each command that was seen on the setup script command line. - Uses the list of commands found and cache of command objects - created by 'get_command_obj()'. - """ - for cmd in self.commands: - self.run_command(cmd) - - # -- Methods that operate on its Commands -------------------------- - - def run_command(self, command): - """Do whatever it takes to run a command (including nothing at all, - if the command has already been run). Specifically: if we have - already created and run the command named by 'command', return - silently without doing anything. If the command named by 'command' - doesn't even have a command object yet, create one. Then invoke - 'run()' on that command object (or an existing one). - """ - # Already been here, done that? then return silently. - if self.have_run.get(command): - return - - log.info("running %s", command) - cmd_obj = self.get_command_obj(command) - cmd_obj.ensure_finalized() - cmd_obj.run() - self.have_run[command] = 1 - - # -- Distribution query methods ------------------------------------ - - def has_pure_modules(self): - return len(self.packages or self.py_modules or []) > 0 - - def has_ext_modules(self): - return self.ext_modules and len(self.ext_modules) > 0 - - def has_c_libraries(self): - return self.libraries and len(self.libraries) > 0 - - def has_modules(self): - return self.has_pure_modules() or self.has_ext_modules() - - def has_headers(self): - return self.headers and len(self.headers) > 0 - - def has_scripts(self): - return self.scripts and len(self.scripts) > 0 - - def has_data_files(self): - return self.data_files and len(self.data_files) > 0 - - def is_pure(self): - return (self.has_pure_modules() and - not self.has_ext_modules() and - not self.has_c_libraries()) - - # -- Metadata query methods ---------------------------------------- - - # If you're looking for 'get_name()', 'get_version()', and so forth, - # they are defined in a sneaky way: the constructor binds self.get_XXX - # to self.metadata.get_XXX. The actual code is in the - # DistributionMetadata class, below. - -class DistributionMetadata: - """Dummy class to hold the distribution meta-data: name, version, - author, and so forth. - """ - - _METHOD_BASENAMES = ("name", "version", "author", "author_email", - "maintainer", "maintainer_email", "url", - "license", "description", "long_description", - "keywords", "platforms", "fullname", "contact", - "contact_email", "classifiers", "download_url", - # PEP 314 - "provides", "requires", "obsoletes", - ) - - def __init__(self, path=None): - if path is not None: - self.read_pkg_file(open(path)) - else: - self.name = None - self.version = None - self.author = None - self.author_email = None - self.maintainer = None - self.maintainer_email = None - self.url = None - self.license = None - self.description = None - self.long_description = None - self.keywords = None - self.platforms = None - self.classifiers = None - self.download_url = None - # PEP 314 - self.provides = None - self.requires = None - self.obsoletes = None - - def read_pkg_file(self, file): - """Reads the metadata values from a file object.""" - msg = message_from_file(file) - - def _read_field(name): - value = msg[name] - if value == 'UNKNOWN': - return None - return value - - def _read_list(name): - values = msg.get_all(name, None) - if values == []: - return None - return values - - metadata_version = msg['metadata-version'] - self.name = _read_field('name') - self.version = _read_field('version') - self.description = _read_field('summary') - # we are filling author only. - self.author = _read_field('author') - self.maintainer = None - self.author_email = _read_field('author-email') - self.maintainer_email = None - self.url = _read_field('home-page') - self.license = _read_field('license') - - if 'download-url' in msg: - self.download_url = _read_field('download-url') - else: - self.download_url = None - - self.long_description = _read_field('description') - self.description = _read_field('summary') - - if 'keywords' in msg: - self.keywords = _read_field('keywords').split(',') - - self.platforms = _read_list('platform') - self.classifiers = _read_list('classifier') - - # PEP 314 - these fields only exist in 1.1 - if metadata_version == '1.1': - self.requires = _read_list('requires') - self.provides = _read_list('provides') - self.obsoletes = _read_list('obsoletes') - else: - self.requires = None - self.provides = None - self.obsoletes = None - - def write_pkg_info(self, base_dir): - """Write the PKG-INFO file into the release tree. - """ - with open(os.path.join(base_dir, 'PKG-INFO'), 'w', - encoding='UTF-8') as pkg_info: - self.write_pkg_file(pkg_info) - - def write_pkg_file(self, file): - """Write the PKG-INFO format data to a file object. - """ - version = '1.0' - if (self.provides or self.requires or self.obsoletes or - self.classifiers or self.download_url): - version = '1.1' - - file.write('Metadata-Version: %s\n' % version) - file.write('Name: %s\n' % self.get_name()) - file.write('Version: %s\n' % self.get_version()) - file.write('Summary: %s\n' % self.get_description()) - file.write('Home-page: %s\n' % self.get_url()) - file.write('Author: %s\n' % self.get_contact()) - file.write('Author-email: %s\n' % self.get_contact_email()) - file.write('License: %s\n' % self.get_license()) - if self.download_url: - file.write('Download-URL: %s\n' % self.download_url) - - long_desc = rfc822_escape(self.get_long_description()) - file.write('Description: %s\n' % long_desc) - - keywords = ','.join(self.get_keywords()) - if keywords: - file.write('Keywords: %s\n' % keywords) - - self._write_list(file, 'Platform', self.get_platforms()) - self._write_list(file, 'Classifier', self.get_classifiers()) - - # PEP 314 - self._write_list(file, 'Requires', self.get_requires()) - self._write_list(file, 'Provides', self.get_provides()) - self._write_list(file, 'Obsoletes', self.get_obsoletes()) - - def _write_list(self, file, name, values): - for value in values: - file.write('%s: %s\n' % (name, value)) - - # -- Metadata query methods ---------------------------------------- - - def get_name(self): - return self.name or "UNKNOWN" - - def get_version(self): - return self.version or "0.0.0" - - def get_fullname(self): - return "%s-%s" % (self.get_name(), self.get_version()) - - def get_author(self): - return self.author or "UNKNOWN" - - def get_author_email(self): - return self.author_email or "UNKNOWN" - - def get_maintainer(self): - return self.maintainer or "UNKNOWN" - - def get_maintainer_email(self): - return self.maintainer_email or "UNKNOWN" - - def get_contact(self): - return self.maintainer or self.author or "UNKNOWN" - - def get_contact_email(self): - return self.maintainer_email or self.author_email or "UNKNOWN" - - def get_url(self): - return self.url or "UNKNOWN" - - def get_license(self): - return self.license or "UNKNOWN" - get_licence = get_license - - def get_description(self): - return self.description or "UNKNOWN" - - def get_long_description(self): - return self.long_description or "UNKNOWN" - - def get_keywords(self): - return self.keywords or [] - - def get_platforms(self): - return self.platforms or ["UNKNOWN"] - - def get_classifiers(self): - return self.classifiers or [] - - def get_download_url(self): - return self.download_url or "UNKNOWN" - - # PEP 314 - def get_requires(self): - return self.requires or [] - - def set_requires(self, value): - import distutils.versionpredicate - for v in value: - distutils.versionpredicate.VersionPredicate(v) - self.requires = value - - def get_provides(self): - return self.provides or [] - - def set_provides(self, value): - value = [v.strip() for v in value] - for v in value: - import distutils.versionpredicate - distutils.versionpredicate.split_provision(v) - self.provides = value - - def get_obsoletes(self): - return self.obsoletes or [] - - def set_obsoletes(self, value): - import distutils.versionpredicate - for v in value: - distutils.versionpredicate.VersionPredicate(v) - self.obsoletes = value - -def fix_help_options(options): - """Convert a 4-tuple 'help_options' list as found in various command - classes to the 3-tuple form required by FancyGetopt. - """ - new_options = [] - for help_tuple in options: - new_options.append(help_tuple[0:3]) - return new_options diff --git a/Lib/distutils/errors.py b/Lib/distutils/errors.py deleted file mode 100644 index 8b93059e19f..00000000000 --- a/Lib/distutils/errors.py +++ /dev/null @@ -1,97 +0,0 @@ -"""distutils.errors - -Provides exceptions used by the Distutils modules. Note that Distutils -modules may raise standard exceptions; in particular, SystemExit is -usually raised for errors that are obviously the end-user's fault -(eg. bad command-line arguments). - -This module is safe to use in "from ... import *" mode; it only exports -symbols whose names start with "Distutils" and end with "Error".""" - -class DistutilsError (Exception): - """The root of all Distutils evil.""" - pass - -class DistutilsModuleError (DistutilsError): - """Unable to load an expected module, or to find an expected class - within some module (in particular, command modules and classes).""" - pass - -class DistutilsClassError (DistutilsError): - """Some command class (or possibly distribution class, if anyone - feels a need to subclass Distribution) is found not to be holding - up its end of the bargain, ie. implementing some part of the - "command "interface.""" - pass - -class DistutilsGetoptError (DistutilsError): - """The option table provided to 'fancy_getopt()' is bogus.""" - pass - -class DistutilsArgError (DistutilsError): - """Raised by fancy_getopt in response to getopt.error -- ie. an - error in the command line usage.""" - pass - -class DistutilsFileError (DistutilsError): - """Any problems in the filesystem: expected file not found, etc. - Typically this is for problems that we detect before OSError - could be raised.""" - pass - -class DistutilsOptionError (DistutilsError): - """Syntactic/semantic errors in command options, such as use of - mutually conflicting options, or inconsistent options, - badly-spelled values, etc. No distinction is made between option - values originating in the setup script, the command line, config - files, or what-have-you -- but if we *know* something originated in - the setup script, we'll raise DistutilsSetupError instead.""" - pass - -class DistutilsSetupError (DistutilsError): - """For errors that can be definitely blamed on the setup script, - such as invalid keyword arguments to 'setup()'.""" - pass - -class DistutilsPlatformError (DistutilsError): - """We don't know how to do something on the current platform (but - we do know how to do it on some platform) -- eg. trying to compile - C files on a platform not supported by a CCompiler subclass.""" - pass - -class DistutilsExecError (DistutilsError): - """Any problems executing an external program (such as the C - compiler, when compiling C files).""" - pass - -class DistutilsInternalError (DistutilsError): - """Internal inconsistencies or impossibilities (obviously, this - should never be seen if the code is working!).""" - pass - -class DistutilsTemplateError (DistutilsError): - """Syntax error in a file list template.""" - -class DistutilsByteCompileError(DistutilsError): - """Byte compile error.""" - -# Exception classes used by the CCompiler implementation classes -class CCompilerError (Exception): - """Some compile/link operation failed.""" - -class PreprocessError (CCompilerError): - """Failure to preprocess one or more C/C++ files.""" - -class CompileError (CCompilerError): - """Failure to compile one or more C/C++ source files.""" - -class LibError (CCompilerError): - """Failure to create a static library from one or more C/C++ object - files.""" - -class LinkError (CCompilerError): - """Failure to link one or more C/C++ object files into an executable - or shared library file.""" - -class UnknownFileError (CCompilerError): - """Attempt to process an unknown file type.""" diff --git a/Lib/distutils/extension.py b/Lib/distutils/extension.py deleted file mode 100644 index c507da360aa..00000000000 --- a/Lib/distutils/extension.py +++ /dev/null @@ -1,240 +0,0 @@ -"""distutils.extension - -Provides the Extension class, used to describe C/C++ extension -modules in setup scripts.""" - -import os -import warnings - -# This class is really only used by the "build_ext" command, so it might -# make sense to put it in distutils.command.build_ext. However, that -# module is already big enough, and I want to make this class a bit more -# complex to simplify some common cases ("foo" module in "foo.c") and do -# better error-checking ("foo.c" actually exists). -# -# Also, putting this in build_ext.py means every setup script would have to -# import that large-ish module (indirectly, through distutils.core) in -# order to do anything. - -class Extension: - """Just a collection of attributes that describes an extension - module and everything needed to build it (hopefully in a portable - way, but there are hooks that let you be as unportable as you need). - - Instance attributes: - name : string - the full name of the extension, including any packages -- ie. - *not* a filename or pathname, but Python dotted name - sources : [string] - list of source filenames, relative to the distribution root - (where the setup script lives), in Unix form (slash-separated) - for portability. Source files may be C, C++, SWIG (.i), - platform-specific resource files, or whatever else is recognized - by the "build_ext" command as source for a Python extension. - include_dirs : [string] - list of directories to search for C/C++ header files (in Unix - form for portability) - define_macros : [(name : string, value : string|None)] - list of macros to define; each macro is defined using a 2-tuple, - where 'value' is either the string to define it to or None to - define it without a particular value (equivalent of "#define - FOO" in source or -DFOO on Unix C compiler command line) - undef_macros : [string] - list of macros to undefine explicitly - library_dirs : [string] - list of directories to search for C/C++ libraries at link time - libraries : [string] - list of library names (not filenames or paths) to link against - runtime_library_dirs : [string] - list of directories to search for C/C++ libraries at run time - (for shared extensions, this is when the extension is loaded) - extra_objects : [string] - list of extra files to link with (eg. object files not implied - by 'sources', static library that must be explicitly specified, - binary resource files, etc.) - extra_compile_args : [string] - any extra platform- and compiler-specific information to use - when compiling the source files in 'sources'. For platforms and - compilers where "command line" makes sense, this is typically a - list of command-line arguments, but for other platforms it could - be anything. - extra_link_args : [string] - any extra platform- and compiler-specific information to use - when linking object files together to create the extension (or - to create a new static Python interpreter). Similar - interpretation as for 'extra_compile_args'. - export_symbols : [string] - list of symbols to be exported from a shared extension. Not - used on all platforms, and not generally necessary for Python - extensions, which typically export exactly one symbol: "init" + - extension_name. - swig_opts : [string] - any extra options to pass to SWIG if a source file has the .i - extension. - depends : [string] - list of files that the extension depends on - language : string - extension language (i.e. "c", "c++", "objc"). Will be detected - from the source extensions if not provided. - optional : boolean - specifies that a build failure in the extension should not abort the - build process, but simply not install the failing extension. - """ - - # When adding arguments to this constructor, be sure to update - # setup_keywords in core.py. - def __init__(self, name, sources, - include_dirs=None, - define_macros=None, - undef_macros=None, - library_dirs=None, - libraries=None, - runtime_library_dirs=None, - extra_objects=None, - extra_compile_args=None, - extra_link_args=None, - export_symbols=None, - swig_opts = None, - depends=None, - language=None, - optional=None, - **kw # To catch unknown keywords - ): - if not isinstance(name, str): - raise AssertionError("'name' must be a string") - if not (isinstance(sources, list) and - all(isinstance(v, str) for v in sources)): - raise AssertionError("'sources' must be a list of strings") - - self.name = name - self.sources = sources - self.include_dirs = include_dirs or [] - self.define_macros = define_macros or [] - self.undef_macros = undef_macros or [] - self.library_dirs = library_dirs or [] - self.libraries = libraries or [] - self.runtime_library_dirs = runtime_library_dirs or [] - self.extra_objects = extra_objects or [] - self.extra_compile_args = extra_compile_args or [] - self.extra_link_args = extra_link_args or [] - self.export_symbols = export_symbols or [] - self.swig_opts = swig_opts or [] - self.depends = depends or [] - self.language = language - self.optional = optional - - # If there are unknown keyword options, warn about them - if len(kw) > 0: - options = [repr(option) for option in kw] - options = ', '.join(sorted(options)) - msg = "Unknown Extension options: %s" % options - warnings.warn(msg) - - def __repr__(self): - return '<%s.%s(%r) at %#x>' % ( - self.__class__.__module__, - self.__class__.__qualname__, - self.name, - id(self)) - - -def read_setup_file(filename): - """Reads a Setup file and returns Extension instances.""" - from distutils.sysconfig import (parse_makefile, expand_makefile_vars, - _variable_rx) - - from distutils.text_file import TextFile - from distutils.util import split_quoted - - # First pass over the file to gather "VAR = VALUE" assignments. - vars = parse_makefile(filename) - - # Second pass to gobble up the real content: lines of the form - # ... [ ...] [ ...] [ ...] - file = TextFile(filename, - strip_comments=1, skip_blanks=1, join_lines=1, - lstrip_ws=1, rstrip_ws=1) - try: - extensions = [] - - while True: - line = file.readline() - if line is None: # eof - break - if _variable_rx.match(line): # VAR=VALUE, handled in first pass - continue - - if line[0] == line[-1] == "*": - file.warn("'%s' lines not handled yet" % line) - continue - - line = expand_makefile_vars(line, vars) - words = split_quoted(line) - - # NB. this parses a slightly different syntax than the old - # makesetup script: here, there must be exactly one extension per - # line, and it must be the first word of the line. I have no idea - # why the old syntax supported multiple extensions per line, as - # they all wind up being the same. - - module = words[0] - ext = Extension(module, []) - append_next_word = None - - for word in words[1:]: - if append_next_word is not None: - append_next_word.append(word) - append_next_word = None - continue - - suffix = os.path.splitext(word)[1] - switch = word[0:2] ; value = word[2:] - - if suffix in (".c", ".cc", ".cpp", ".cxx", ".c++", ".m", ".mm"): - # hmm, should we do something about C vs. C++ sources? - # or leave it up to the CCompiler implementation to - # worry about? - ext.sources.append(word) - elif switch == "-I": - ext.include_dirs.append(value) - elif switch == "-D": - equals = value.find("=") - if equals == -1: # bare "-DFOO" -- no value - ext.define_macros.append((value, None)) - else: # "-DFOO=blah" - ext.define_macros.append((value[0:equals], - value[equals+2:])) - elif switch == "-U": - ext.undef_macros.append(value) - elif switch == "-C": # only here 'cause makesetup has it! - ext.extra_compile_args.append(word) - elif switch == "-l": - ext.libraries.append(value) - elif switch == "-L": - ext.library_dirs.append(value) - elif switch == "-R": - ext.runtime_library_dirs.append(value) - elif word == "-rpath": - append_next_word = ext.runtime_library_dirs - elif word == "-Xlinker": - append_next_word = ext.extra_link_args - elif word == "-Xcompiler": - append_next_word = ext.extra_compile_args - elif switch == "-u": - ext.extra_link_args.append(word) - if not value: - append_next_word = ext.extra_link_args - elif suffix in (".a", ".so", ".sl", ".o", ".dylib"): - # NB. a really faithful emulation of makesetup would - # append a .o file to extra_objects only if it - # had a slash in it; otherwise, it would s/.o/.c/ - # and append it to sources. Hmmmm. - ext.extra_objects.append(word) - else: - file.warn("unrecognized argument '%s'" % word) - - extensions.append(ext) - finally: - file.close() - - return extensions diff --git a/Lib/distutils/fancy_getopt.py b/Lib/distutils/fancy_getopt.py deleted file mode 100644 index 7d170dd2773..00000000000 --- a/Lib/distutils/fancy_getopt.py +++ /dev/null @@ -1,457 +0,0 @@ -"""distutils.fancy_getopt - -Wrapper around the standard getopt module that provides the following -additional features: - * short and long options are tied together - * options have help strings, so fancy_getopt could potentially - create a complete usage summary - * options set attributes of a passed-in object -""" - -import sys, string, re -import getopt -from distutils.errors import * - -# Much like command_re in distutils.core, this is close to but not quite -# the same as a Python NAME -- except, in the spirit of most GNU -# utilities, we use '-' in place of '_'. (The spirit of LISP lives on!) -# The similarities to NAME are again not a coincidence... -longopt_pat = r'[a-zA-Z](?:[a-zA-Z0-9-]*)' -longopt_re = re.compile(r'^%s$' % longopt_pat) - -# For recognizing "negative alias" options, eg. "quiet=!verbose" -neg_alias_re = re.compile("^(%s)=!(%s)$" % (longopt_pat, longopt_pat)) - -# This is used to translate long options to legitimate Python identifiers -# (for use as attributes of some object). -longopt_xlate = str.maketrans('-', '_') - -class FancyGetopt: - """Wrapper around the standard 'getopt()' module that provides some - handy extra functionality: - * short and long options are tied together - * options have help strings, and help text can be assembled - from them - * options set attributes of a passed-in object - * boolean options can have "negative aliases" -- eg. if - --quiet is the "negative alias" of --verbose, then "--quiet" - on the command line sets 'verbose' to false - """ - - def __init__(self, option_table=None): - # The option table is (currently) a list of tuples. The - # tuples may have 3 or four values: - # (long_option, short_option, help_string [, repeatable]) - # if an option takes an argument, its long_option should have '=' - # appended; short_option should just be a single character, no ':' - # in any case. If a long_option doesn't have a corresponding - # short_option, short_option should be None. All option tuples - # must have long options. - self.option_table = option_table - - # 'option_index' maps long option names to entries in the option - # table (ie. those 3-tuples). - self.option_index = {} - if self.option_table: - self._build_index() - - # 'alias' records (duh) alias options; {'foo': 'bar'} means - # --foo is an alias for --bar - self.alias = {} - - # 'negative_alias' keeps track of options that are the boolean - # opposite of some other option - self.negative_alias = {} - - # These keep track of the information in the option table. We - # don't actually populate these structures until we're ready to - # parse the command-line, since the 'option_table' passed in here - # isn't necessarily the final word. - self.short_opts = [] - self.long_opts = [] - self.short2long = {} - self.attr_name = {} - self.takes_arg = {} - - # And 'option_order' is filled up in 'getopt()'; it records the - # original order of options (and their values) on the command-line, - # but expands short options, converts aliases, etc. - self.option_order = [] - - def _build_index(self): - self.option_index.clear() - for option in self.option_table: - self.option_index[option[0]] = option - - def set_option_table(self, option_table): - self.option_table = option_table - self._build_index() - - def add_option(self, long_option, short_option=None, help_string=None): - if long_option in self.option_index: - raise DistutilsGetoptError( - "option conflict: already an option '%s'" % long_option) - else: - option = (long_option, short_option, help_string) - self.option_table.append(option) - self.option_index[long_option] = option - - def has_option(self, long_option): - """Return true if the option table for this parser has an - option with long name 'long_option'.""" - return long_option in self.option_index - - def get_attr_name(self, long_option): - """Translate long option name 'long_option' to the form it - has as an attribute of some object: ie., translate hyphens - to underscores.""" - return long_option.translate(longopt_xlate) - - def _check_alias_dict(self, aliases, what): - assert isinstance(aliases, dict) - for (alias, opt) in aliases.items(): - if alias not in self.option_index: - raise DistutilsGetoptError(("invalid %s '%s': " - "option '%s' not defined") % (what, alias, alias)) - if opt not in self.option_index: - raise DistutilsGetoptError(("invalid %s '%s': " - "aliased option '%s' not defined") % (what, alias, opt)) - - def set_aliases(self, alias): - """Set the aliases for this option parser.""" - self._check_alias_dict(alias, "alias") - self.alias = alias - - def set_negative_aliases(self, negative_alias): - """Set the negative aliases for this option parser. - 'negative_alias' should be a dictionary mapping option names to - option names, both the key and value must already be defined - in the option table.""" - self._check_alias_dict(negative_alias, "negative alias") - self.negative_alias = negative_alias - - def _grok_option_table(self): - """Populate the various data structures that keep tabs on the - option table. Called by 'getopt()' before it can do anything - worthwhile. - """ - self.long_opts = [] - self.short_opts = [] - self.short2long.clear() - self.repeat = {} - - for option in self.option_table: - if len(option) == 3: - long, short, help = option - repeat = 0 - elif len(option) == 4: - long, short, help, repeat = option - else: - # the option table is part of the code, so simply - # assert that it is correct - raise ValueError("invalid option tuple: %r" % (option,)) - - # Type- and value-check the option names - if not isinstance(long, str) or len(long) < 2: - raise DistutilsGetoptError(("invalid long option '%s': " - "must be a string of length >= 2") % long) - - if (not ((short is None) or - (isinstance(short, str) and len(short) == 1))): - raise DistutilsGetoptError("invalid short option '%s': " - "must a single character or None" % short) - - self.repeat[long] = repeat - self.long_opts.append(long) - - if long[-1] == '=': # option takes an argument? - if short: short = short + ':' - long = long[0:-1] - self.takes_arg[long] = 1 - else: - # Is option is a "negative alias" for some other option (eg. - # "quiet" == "!verbose")? - alias_to = self.negative_alias.get(long) - if alias_to is not None: - if self.takes_arg[alias_to]: - raise DistutilsGetoptError( - "invalid negative alias '%s': " - "aliased option '%s' takes a value" - % (long, alias_to)) - - self.long_opts[-1] = long # XXX redundant?! - self.takes_arg[long] = 0 - - # If this is an alias option, make sure its "takes arg" flag is - # the same as the option it's aliased to. - alias_to = self.alias.get(long) - if alias_to is not None: - if self.takes_arg[long] != self.takes_arg[alias_to]: - raise DistutilsGetoptError( - "invalid alias '%s': inconsistent with " - "aliased option '%s' (one of them takes a value, " - "the other doesn't" - % (long, alias_to)) - - # Now enforce some bondage on the long option name, so we can - # later translate it to an attribute name on some object. Have - # to do this a bit late to make sure we've removed any trailing - # '='. - if not longopt_re.match(long): - raise DistutilsGetoptError( - "invalid long option name '%s' " - "(must be letters, numbers, hyphens only" % long) - - self.attr_name[long] = self.get_attr_name(long) - if short: - self.short_opts.append(short) - self.short2long[short[0]] = long - - def getopt(self, args=None, object=None): - """Parse command-line options in args. Store as attributes on object. - - If 'args' is None or not supplied, uses 'sys.argv[1:]'. If - 'object' is None or not supplied, creates a new OptionDummy - object, stores option values there, and returns a tuple (args, - object). If 'object' is supplied, it is modified in place and - 'getopt()' just returns 'args'; in both cases, the returned - 'args' is a modified copy of the passed-in 'args' list, which - is left untouched. - """ - if args is None: - args = sys.argv[1:] - if object is None: - object = OptionDummy() - created_object = True - else: - created_object = False - - self._grok_option_table() - - short_opts = ' '.join(self.short_opts) - try: - opts, args = getopt.getopt(args, short_opts, self.long_opts) - except getopt.error as msg: - raise DistutilsArgError(msg) - - for opt, val in opts: - if len(opt) == 2 and opt[0] == '-': # it's a short option - opt = self.short2long[opt[1]] - else: - assert len(opt) > 2 and opt[:2] == '--' - opt = opt[2:] - - alias = self.alias.get(opt) - if alias: - opt = alias - - if not self.takes_arg[opt]: # boolean option? - assert val == '', "boolean option can't have value" - alias = self.negative_alias.get(opt) - if alias: - opt = alias - val = 0 - else: - val = 1 - - attr = self.attr_name[opt] - # The only repeating option at the moment is 'verbose'. - # It has a negative option -q quiet, which should set verbose = 0. - if val and self.repeat.get(attr) is not None: - val = getattr(object, attr, 0) + 1 - setattr(object, attr, val) - self.option_order.append((opt, val)) - - # for opts - if created_object: - return args, object - else: - return args - - def get_option_order(self): - """Returns the list of (option, value) tuples processed by the - previous run of 'getopt()'. Raises RuntimeError if - 'getopt()' hasn't been called yet. - """ - if self.option_order is None: - raise RuntimeError("'getopt()' hasn't been called yet") - else: - return self.option_order - - def generate_help(self, header=None): - """Generate help text (a list of strings, one per suggested line of - output) from the option table for this FancyGetopt object. - """ - # Blithely assume the option table is good: probably wouldn't call - # 'generate_help()' unless you've already called 'getopt()'. - - # First pass: determine maximum length of long option names - max_opt = 0 - for option in self.option_table: - long = option[0] - short = option[1] - l = len(long) - if long[-1] == '=': - l = l - 1 - if short is not None: - l = l + 5 # " (-x)" where short == 'x' - if l > max_opt: - max_opt = l - - opt_width = max_opt + 2 + 2 + 2 # room for indent + dashes + gutter - - # Typical help block looks like this: - # --foo controls foonabulation - # Help block for longest option looks like this: - # --flimflam set the flim-flam level - # and with wrapped text: - # --flimflam set the flim-flam level (must be between - # 0 and 100, except on Tuesdays) - # Options with short names will have the short name shown (but - # it doesn't contribute to max_opt): - # --foo (-f) controls foonabulation - # If adding the short option would make the left column too wide, - # we push the explanation off to the next line - # --flimflam (-l) - # set the flim-flam level - # Important parameters: - # - 2 spaces before option block start lines - # - 2 dashes for each long option name - # - min. 2 spaces between option and explanation (gutter) - # - 5 characters (incl. space) for short option name - - # Now generate lines of help text. (If 80 columns were good enough - # for Jesus, then 78 columns are good enough for me!) - line_width = 78 - text_width = line_width - opt_width - big_indent = ' ' * opt_width - if header: - lines = [header] - else: - lines = ['Option summary:'] - - for option in self.option_table: - long, short, help = option[:3] - text = wrap_text(help, text_width) - if long[-1] == '=': - long = long[0:-1] - - # Case 1: no short option at all (makes life easy) - if short is None: - if text: - lines.append(" --%-*s %s" % (max_opt, long, text[0])) - else: - lines.append(" --%-*s " % (max_opt, long)) - - # Case 2: we have a short option, so we have to include it - # just after the long option - else: - opt_names = "%s (-%s)" % (long, short) - if text: - lines.append(" --%-*s %s" % - (max_opt, opt_names, text[0])) - else: - lines.append(" --%-*s" % opt_names) - - for l in text[1:]: - lines.append(big_indent + l) - return lines - - def print_help(self, header=None, file=None): - if file is None: - file = sys.stdout - for line in self.generate_help(header): - file.write(line + "\n") - - -def fancy_getopt(options, negative_opt, object, args): - parser = FancyGetopt(options) - parser.set_negative_aliases(negative_opt) - return parser.getopt(args, object) - - -WS_TRANS = {ord(_wschar) : ' ' for _wschar in string.whitespace} - -def wrap_text(text, width): - """wrap_text(text : string, width : int) -> [string] - - Split 'text' into multiple lines of no more than 'width' characters - each, and return the list of strings that results. - """ - if text is None: - return [] - if len(text) <= width: - return [text] - - text = text.expandtabs() - text = text.translate(WS_TRANS) - chunks = re.split(r'( +|-+)', text) - chunks = [ch for ch in chunks if ch] # ' - ' results in empty strings - lines = [] - - while chunks: - cur_line = [] # list of chunks (to-be-joined) - cur_len = 0 # length of current line - - while chunks: - l = len(chunks[0]) - if cur_len + l <= width: # can squeeze (at least) this chunk in - cur_line.append(chunks[0]) - del chunks[0] - cur_len = cur_len + l - else: # this line is full - # drop last chunk if all space - if cur_line and cur_line[-1][0] == ' ': - del cur_line[-1] - break - - if chunks: # any chunks left to process? - # if the current line is still empty, then we had a single - # chunk that's too big too fit on a line -- so we break - # down and break it up at the line width - if cur_len == 0: - cur_line.append(chunks[0][0:width]) - chunks[0] = chunks[0][width:] - - # all-whitespace chunks at the end of a line can be discarded - # (and we know from the re.split above that if a chunk has - # *any* whitespace, it is *all* whitespace) - if chunks[0][0] == ' ': - del chunks[0] - - # and store this line in the list-of-all-lines -- as a single - # string, of course! - lines.append(''.join(cur_line)) - - return lines - - -def translate_longopt(opt): - """Convert a long option name to a valid Python identifier by - changing "-" to "_". - """ - return opt.translate(longopt_xlate) - - -class OptionDummy: - """Dummy class just used as a place to hold command-line option - values as instance attributes.""" - - def __init__(self, options=[]): - """Create a new OptionDummy instance. The attributes listed in - 'options' will be initialized to None.""" - for opt in options: - setattr(self, opt, None) - - -if __name__ == "__main__": - text = """\ -Tra-la-la, supercalifragilisticexpialidocious. -How *do* you spell that odd word, anyways? -(Someone ask Mary -- she'll know [or she'll -say, "How should I know?"].)""" - - for w in (10, 20, 30, 40): - print("width: %d" % w) - print("\n".join(wrap_text(text, w))) - print() diff --git a/Lib/distutils/file_util.py b/Lib/distutils/file_util.py deleted file mode 100644 index b3fee35a6cc..00000000000 --- a/Lib/distutils/file_util.py +++ /dev/null @@ -1,238 +0,0 @@ -"""distutils.file_util - -Utility functions for operating on single files. -""" - -import os -from distutils.errors import DistutilsFileError -from distutils import log - -# for generating verbose output in 'copy_file()' -_copy_action = { None: 'copying', - 'hard': 'hard linking', - 'sym': 'symbolically linking' } - - -def _copy_file_contents(src, dst, buffer_size=16*1024): - """Copy the file 'src' to 'dst'; both must be filenames. Any error - opening either file, reading from 'src', or writing to 'dst', raises - DistutilsFileError. Data is read/written in chunks of 'buffer_size' - bytes (default 16k). No attempt is made to handle anything apart from - regular files. - """ - # Stolen from shutil module in the standard library, but with - # custom error-handling added. - fsrc = None - fdst = None - try: - try: - fsrc = open(src, 'rb') - except OSError as e: - raise DistutilsFileError("could not open '%s': %s" % (src, e.strerror)) - - if os.path.exists(dst): - try: - os.unlink(dst) - except OSError as e: - raise DistutilsFileError( - "could not delete '%s': %s" % (dst, e.strerror)) - - try: - fdst = open(dst, 'wb') - except OSError as e: - raise DistutilsFileError( - "could not create '%s': %s" % (dst, e.strerror)) - - while True: - try: - buf = fsrc.read(buffer_size) - except OSError as e: - raise DistutilsFileError( - "could not read from '%s': %s" % (src, e.strerror)) - - if not buf: - break - - try: - fdst.write(buf) - except OSError as e: - raise DistutilsFileError( - "could not write to '%s': %s" % (dst, e.strerror)) - finally: - if fdst: - fdst.close() - if fsrc: - fsrc.close() - -def copy_file(src, dst, preserve_mode=1, preserve_times=1, update=0, - link=None, verbose=1, dry_run=0): - """Copy a file 'src' to 'dst'. If 'dst' is a directory, then 'src' is - copied there with the same name; otherwise, it must be a filename. (If - the file exists, it will be ruthlessly clobbered.) If 'preserve_mode' - is true (the default), the file's mode (type and permission bits, or - whatever is analogous on the current platform) is copied. If - 'preserve_times' is true (the default), the last-modified and - last-access times are copied as well. If 'update' is true, 'src' will - only be copied if 'dst' does not exist, or if 'dst' does exist but is - older than 'src'. - - 'link' allows you to make hard links (os.link) or symbolic links - (os.symlink) instead of copying: set it to "hard" or "sym"; if it is - None (the default), files are copied. Don't set 'link' on systems that - don't support it: 'copy_file()' doesn't check if hard or symbolic - linking is available. If hardlink fails, falls back to - _copy_file_contents(). - - Under Mac OS, uses the native file copy function in macostools; on - other systems, uses '_copy_file_contents()' to copy file contents. - - Return a tuple (dest_name, copied): 'dest_name' is the actual name of - the output file, and 'copied' is true if the file was copied (or would - have been copied, if 'dry_run' true). - """ - # XXX if the destination file already exists, we clobber it if - # copying, but blow up if linking. Hmmm. And I don't know what - # macostools.copyfile() does. Should definitely be consistent, and - # should probably blow up if destination exists and we would be - # changing it (ie. it's not already a hard/soft link to src OR - # (not update) and (src newer than dst). - - from distutils.dep_util import newer - from stat import ST_ATIME, ST_MTIME, ST_MODE, S_IMODE - - if not os.path.isfile(src): - raise DistutilsFileError( - "can't copy '%s': doesn't exist or not a regular file" % src) - - if os.path.isdir(dst): - dir = dst - dst = os.path.join(dst, os.path.basename(src)) - else: - dir = os.path.dirname(dst) - - if update and not newer(src, dst): - if verbose >= 1: - log.debug("not copying %s (output up-to-date)", src) - return (dst, 0) - - try: - action = _copy_action[link] - except KeyError: - raise ValueError("invalid value '%s' for 'link' argument" % link) - - if verbose >= 1: - if os.path.basename(dst) == os.path.basename(src): - log.info("%s %s -> %s", action, src, dir) - else: - log.info("%s %s -> %s", action, src, dst) - - if dry_run: - return (dst, 1) - - # If linking (hard or symbolic), use the appropriate system call - # (Unix only, of course, but that's the caller's responsibility) - elif link == 'hard': - if not (os.path.exists(dst) and os.path.samefile(src, dst)): - try: - os.link(src, dst) - return (dst, 1) - except OSError: - # If hard linking fails, fall back on copying file - # (some special filesystems don't support hard linking - # even under Unix, see issue #8876). - pass - elif link == 'sym': - if not (os.path.exists(dst) and os.path.samefile(src, dst)): - os.symlink(src, dst) - return (dst, 1) - - # Otherwise (non-Mac, not linking), copy the file contents and - # (optionally) copy the times and mode. - _copy_file_contents(src, dst) - if preserve_mode or preserve_times: - st = os.stat(src) - - # According to David Ascher , utime() should be done - # before chmod() (at least under NT). - if preserve_times: - os.utime(dst, (st[ST_ATIME], st[ST_MTIME])) - if preserve_mode: - os.chmod(dst, S_IMODE(st[ST_MODE])) - - return (dst, 1) - - -# XXX I suspect this is Unix-specific -- need porting help! -def move_file (src, dst, - verbose=1, - dry_run=0): - - """Move a file 'src' to 'dst'. If 'dst' is a directory, the file will - be moved into it with the same name; otherwise, 'src' is just renamed - to 'dst'. Return the new full name of the file. - - Handles cross-device moves on Unix using 'copy_file()'. What about - other systems??? - """ - from os.path import exists, isfile, isdir, basename, dirname - import errno - - if verbose >= 1: - log.info("moving %s -> %s", src, dst) - - if dry_run: - return dst - - if not isfile(src): - raise DistutilsFileError("can't move '%s': not a regular file" % src) - - if isdir(dst): - dst = os.path.join(dst, basename(src)) - elif exists(dst): - raise DistutilsFileError( - "can't move '%s': destination '%s' already exists" % - (src, dst)) - - if not isdir(dirname(dst)): - raise DistutilsFileError( - "can't move '%s': destination '%s' not a valid path" % - (src, dst)) - - copy_it = False - try: - os.rename(src, dst) - except OSError as e: - (num, msg) = e.args - if num == errno.EXDEV: - copy_it = True - else: - raise DistutilsFileError( - "couldn't move '%s' to '%s': %s" % (src, dst, msg)) - - if copy_it: - copy_file(src, dst, verbose=verbose) - try: - os.unlink(src) - except OSError as e: - (num, msg) = e.args - try: - os.unlink(dst) - except OSError: - pass - raise DistutilsFileError( - "couldn't move '%s' to '%s' by copy/delete: " - "delete '%s' failed: %s" - % (src, dst, src, msg)) - return dst - - -def write_file (filename, contents): - """Create a file with the specified name and write 'contents' (a - sequence of strings without line terminators) to it. - """ - f = open(filename, "w") - try: - for line in contents: - f.write(line + "\n") - finally: - f.close() diff --git a/Lib/distutils/filelist.py b/Lib/distutils/filelist.py deleted file mode 100644 index c92d5fdba39..00000000000 --- a/Lib/distutils/filelist.py +++ /dev/null @@ -1,327 +0,0 @@ -"""distutils.filelist - -Provides the FileList class, used for poking about the filesystem -and building lists of files. -""" - -import os, re -import fnmatch -import functools -from distutils.util import convert_path -from distutils.errors import DistutilsTemplateError, DistutilsInternalError -from distutils import log - -class FileList: - """A list of files built by on exploring the filesystem and filtered by - applying various patterns to what we find there. - - Instance attributes: - dir - directory from which files will be taken -- only used if - 'allfiles' not supplied to constructor - files - list of filenames currently being built/filtered/manipulated - allfiles - complete list of files under consideration (ie. without any - filtering applied) - """ - - def __init__(self, warn=None, debug_print=None): - # ignore argument to FileList, but keep them for backwards - # compatibility - self.allfiles = None - self.files = [] - - def set_allfiles(self, allfiles): - self.allfiles = allfiles - - def findall(self, dir=os.curdir): - self.allfiles = findall(dir) - - def debug_print(self, msg): - """Print 'msg' to stdout if the global DEBUG (taken from the - DISTUTILS_DEBUG environment variable) flag is true. - """ - from distutils.debug import DEBUG - if DEBUG: - print(msg) - - # -- List-like methods --------------------------------------------- - - def append(self, item): - self.files.append(item) - - def extend(self, items): - self.files.extend(items) - - def sort(self): - # Not a strict lexical sort! - sortable_files = sorted(map(os.path.split, self.files)) - self.files = [] - for sort_tuple in sortable_files: - self.files.append(os.path.join(*sort_tuple)) - - - # -- Other miscellaneous utility methods --------------------------- - - def remove_duplicates(self): - # Assumes list has been sorted! - for i in range(len(self.files) - 1, 0, -1): - if self.files[i] == self.files[i - 1]: - del self.files[i] - - - # -- "File template" methods --------------------------------------- - - def _parse_template_line(self, line): - words = line.split() - action = words[0] - - patterns = dir = dir_pattern = None - - if action in ('include', 'exclude', - 'global-include', 'global-exclude'): - if len(words) < 2: - raise DistutilsTemplateError( - "'%s' expects ..." % action) - patterns = [convert_path(w) for w in words[1:]] - elif action in ('recursive-include', 'recursive-exclude'): - if len(words) < 3: - raise DistutilsTemplateError( - "'%s' expects

..." % action) - dir = convert_path(words[1]) - patterns = [convert_path(w) for w in words[2:]] - elif action in ('graft', 'prune'): - if len(words) != 2: - raise DistutilsTemplateError( - "'%s' expects a single " % action) - dir_pattern = convert_path(words[1]) - else: - raise DistutilsTemplateError("unknown action '%s'" % action) - - return (action, patterns, dir, dir_pattern) - - def process_template_line(self, line): - # Parse the line: split it up, make sure the right number of words - # is there, and return the relevant words. 'action' is always - # defined: it's the first word of the line. Which of the other - # three are defined depends on the action; it'll be either - # patterns, (dir and patterns), or (dir_pattern). - (action, patterns, dir, dir_pattern) = self._parse_template_line(line) - - # OK, now we know that the action is valid and we have the - # right number of words on the line for that action -- so we - # can proceed with minimal error-checking. - if action == 'include': - self.debug_print("include " + ' '.join(patterns)) - for pattern in patterns: - if not self.include_pattern(pattern, anchor=1): - log.warn("warning: no files found matching '%s'", - pattern) - - elif action == 'exclude': - self.debug_print("exclude " + ' '.join(patterns)) - for pattern in patterns: - if not self.exclude_pattern(pattern, anchor=1): - log.warn(("warning: no previously-included files " - "found matching '%s'"), pattern) - - elif action == 'global-include': - self.debug_print("global-include " + ' '.join(patterns)) - for pattern in patterns: - if not self.include_pattern(pattern, anchor=0): - log.warn(("warning: no files found matching '%s' " - "anywhere in distribution"), pattern) - - elif action == 'global-exclude': - self.debug_print("global-exclude " + ' '.join(patterns)) - for pattern in patterns: - if not self.exclude_pattern(pattern, anchor=0): - log.warn(("warning: no previously-included files matching " - "'%s' found anywhere in distribution"), - pattern) - - elif action == 'recursive-include': - self.debug_print("recursive-include %s %s" % - (dir, ' '.join(patterns))) - for pattern in patterns: - if not self.include_pattern(pattern, prefix=dir): - log.warn(("warning: no files found matching '%s' " - "under directory '%s'"), - pattern, dir) - - elif action == 'recursive-exclude': - self.debug_print("recursive-exclude %s %s" % - (dir, ' '.join(patterns))) - for pattern in patterns: - if not self.exclude_pattern(pattern, prefix=dir): - log.warn(("warning: no previously-included files matching " - "'%s' found under directory '%s'"), - pattern, dir) - - elif action == 'graft': - self.debug_print("graft " + dir_pattern) - if not self.include_pattern(None, prefix=dir_pattern): - log.warn("warning: no directories found matching '%s'", - dir_pattern) - - elif action == 'prune': - self.debug_print("prune " + dir_pattern) - if not self.exclude_pattern(None, prefix=dir_pattern): - log.warn(("no previously-included directories found " - "matching '%s'"), dir_pattern) - else: - raise DistutilsInternalError( - "this cannot happen: invalid action '%s'" % action) - - - # -- Filtering/selection methods ----------------------------------- - - def include_pattern(self, pattern, anchor=1, prefix=None, is_regex=0): - """Select strings (presumably filenames) from 'self.files' that - match 'pattern', a Unix-style wildcard (glob) pattern. Patterns - are not quite the same as implemented by the 'fnmatch' module: '*' - and '?' match non-special characters, where "special" is platform- - dependent: slash on Unix; colon, slash, and backslash on - DOS/Windows; and colon on Mac OS. - - If 'anchor' is true (the default), then the pattern match is more - stringent: "*.py" will match "foo.py" but not "foo/bar.py". If - 'anchor' is false, both of these will match. - - If 'prefix' is supplied, then only filenames starting with 'prefix' - (itself a pattern) and ending with 'pattern', with anything in between - them, will match. 'anchor' is ignored in this case. - - If 'is_regex' is true, 'anchor' and 'prefix' are ignored, and - 'pattern' is assumed to be either a string containing a regex or a - regex object -- no translation is done, the regex is just compiled - and used as-is. - - Selected strings will be added to self.files. - - Return True if files are found, False otherwise. - """ - # XXX docstring lying about what the special chars are? - files_found = False - pattern_re = translate_pattern(pattern, anchor, prefix, is_regex) - self.debug_print("include_pattern: applying regex r'%s'" % - pattern_re.pattern) - - # delayed loading of allfiles list - if self.allfiles is None: - self.findall() - - for name in self.allfiles: - if pattern_re.search(name): - self.debug_print(" adding " + name) - self.files.append(name) - files_found = True - return files_found - - - def exclude_pattern (self, pattern, - anchor=1, prefix=None, is_regex=0): - """Remove strings (presumably filenames) from 'files' that match - 'pattern'. Other parameters are the same as for - 'include_pattern()', above. - The list 'self.files' is modified in place. - Return True if files are found, False otherwise. - """ - files_found = False - pattern_re = translate_pattern(pattern, anchor, prefix, is_regex) - self.debug_print("exclude_pattern: applying regex r'%s'" % - pattern_re.pattern) - for i in range(len(self.files)-1, -1, -1): - if pattern_re.search(self.files[i]): - self.debug_print(" removing " + self.files[i]) - del self.files[i] - files_found = True - return files_found - - -# ---------------------------------------------------------------------- -# Utility functions - -def _find_all_simple(path): - """ - Find all files under 'path' - """ - results = ( - os.path.join(base, file) - for base, dirs, files in os.walk(path, followlinks=True) - for file in files - ) - return filter(os.path.isfile, results) - - -def findall(dir=os.curdir): - """ - Find all files under 'dir' and return the list of full filenames. - Unless dir is '.', return full filenames with dir prepended. - """ - files = _find_all_simple(dir) - if dir == os.curdir: - make_rel = functools.partial(os.path.relpath, start=dir) - files = map(make_rel, files) - return list(files) - - -def glob_to_re(pattern): - """Translate a shell-like glob pattern to a regular expression; return - a string containing the regex. Differs from 'fnmatch.translate()' in - that '*' does not match "special characters" (which are - platform-specific). - """ - pattern_re = fnmatch.translate(pattern) - - # '?' and '*' in the glob pattern become '.' and '.*' in the RE, which - # IMHO is wrong -- '?' and '*' aren't supposed to match slash in Unix, - # and by extension they shouldn't match such "special characters" under - # any OS. So change all non-escaped dots in the RE to match any - # character except the special characters (currently: just os.sep). - sep = os.sep - if os.sep == '\\': - # we're using a regex to manipulate a regex, so we need - # to escape the backslash twice - sep = r'\\\\' - escaped = r'\1[^%s]' % sep - pattern_re = re.sub(r'((?= self.threshold: - if args: - msg = msg % args - if level in (WARN, ERROR, FATAL): - stream = sys.stderr - else: - stream = sys.stdout - try: - stream.write('%s\n' % msg) - except UnicodeEncodeError: - # emulate backslashreplace error handler - encoding = stream.encoding - msg = msg.encode(encoding, "backslashreplace").decode(encoding) - stream.write('%s\n' % msg) - stream.flush() - - def log(self, level, msg, *args): - self._log(level, msg, args) - - def debug(self, msg, *args): - self._log(DEBUG, msg, args) - - def info(self, msg, *args): - self._log(INFO, msg, args) - - def warn(self, msg, *args): - self._log(WARN, msg, args) - - def error(self, msg, *args): - self._log(ERROR, msg, args) - - def fatal(self, msg, *args): - self._log(FATAL, msg, args) - -_global_log = Log() -log = _global_log.log -debug = _global_log.debug -info = _global_log.info -warn = _global_log.warn -error = _global_log.error -fatal = _global_log.fatal - -def set_threshold(level): - # return the old threshold for use from tests - old = _global_log.threshold - _global_log.threshold = level - return old - -def set_verbosity(v): - if v <= 0: - set_threshold(WARN) - elif v == 1: - set_threshold(INFO) - elif v >= 2: - set_threshold(DEBUG) diff --git a/Lib/distutils/msvc9compiler.py b/Lib/distutils/msvc9compiler.py deleted file mode 100644 index 21191276227..00000000000 --- a/Lib/distutils/msvc9compiler.py +++ /dev/null @@ -1,791 +0,0 @@ -"""distutils.msvc9compiler - -Contains MSVCCompiler, an implementation of the abstract CCompiler class -for the Microsoft Visual Studio 2008. - -The module is compatible with VS 2005 and VS 2008. You can find legacy support -for older versions of VS in distutils.msvccompiler. -""" - -# Written by Perry Stoll -# hacked by Robin Becker and Thomas Heller to do a better job of -# finding DevStudio (through the registry) -# ported to VS2005 and VS 2008 by Christian Heimes - -import os -import subprocess -import sys -import re - -from distutils.errors import DistutilsExecError, DistutilsPlatformError, \ - CompileError, LibError, LinkError -from distutils.ccompiler import CCompiler, gen_preprocess_options, \ - gen_lib_options -from distutils import log -from distutils.util import get_platform - -import winreg - -RegOpenKeyEx = winreg.OpenKeyEx -RegEnumKey = winreg.EnumKey -RegEnumValue = winreg.EnumValue -RegError = winreg.error - -HKEYS = (winreg.HKEY_USERS, - winreg.HKEY_CURRENT_USER, - winreg.HKEY_LOCAL_MACHINE, - winreg.HKEY_CLASSES_ROOT) - -NATIVE_WIN64 = (sys.platform == 'win32' and sys.maxsize > 2**32) -if NATIVE_WIN64: - # Visual C++ is a 32-bit application, so we need to look in - # the corresponding registry branch, if we're running a - # 64-bit Python on Win64 - VS_BASE = r"Software\Wow6432Node\Microsoft\VisualStudio\%0.1f" - WINSDK_BASE = r"Software\Wow6432Node\Microsoft\Microsoft SDKs\Windows" - NET_BASE = r"Software\Wow6432Node\Microsoft\.NETFramework" -else: - VS_BASE = r"Software\Microsoft\VisualStudio\%0.1f" - WINSDK_BASE = r"Software\Microsoft\Microsoft SDKs\Windows" - NET_BASE = r"Software\Microsoft\.NETFramework" - -# A map keyed by get_platform() return values to values accepted by -# 'vcvarsall.bat'. Note a cross-compile may combine these (eg, 'x86_amd64' is -# the param to cross-compile on x86 targeting amd64.) -PLAT_TO_VCVARS = { - 'win32' : 'x86', - 'win-amd64' : 'amd64', - 'win-ia64' : 'ia64', -} - -class Reg: - """Helper class to read values from the registry - """ - - def get_value(cls, path, key): - for base in HKEYS: - d = cls.read_values(base, path) - if d and key in d: - return d[key] - raise KeyError(key) - get_value = classmethod(get_value) - - def read_keys(cls, base, key): - """Return list of registry keys.""" - try: - handle = RegOpenKeyEx(base, key) - except RegError: - return None - L = [] - i = 0 - while True: - try: - k = RegEnumKey(handle, i) - except RegError: - break - L.append(k) - i += 1 - return L - read_keys = classmethod(read_keys) - - def read_values(cls, base, key): - """Return dict of registry keys and values. - - All names are converted to lowercase. - """ - try: - handle = RegOpenKeyEx(base, key) - except RegError: - return None - d = {} - i = 0 - while True: - try: - name, value, type = RegEnumValue(handle, i) - except RegError: - break - name = name.lower() - d[cls.convert_mbcs(name)] = cls.convert_mbcs(value) - i += 1 - return d - read_values = classmethod(read_values) - - def convert_mbcs(s): - dec = getattr(s, "decode", None) - if dec is not None: - try: - s = dec("mbcs") - except UnicodeError: - pass - return s - convert_mbcs = staticmethod(convert_mbcs) - -class MacroExpander: - - def __init__(self, version): - self.macros = {} - self.vsbase = VS_BASE % version - self.load_macros(version) - - def set_macro(self, macro, path, key): - self.macros["$(%s)" % macro] = Reg.get_value(path, key) - - def load_macros(self, version): - self.set_macro("VCInstallDir", self.vsbase + r"\Setup\VC", "productdir") - self.set_macro("VSInstallDir", self.vsbase + r"\Setup\VS", "productdir") - self.set_macro("FrameworkDir", NET_BASE, "installroot") - try: - if version >= 8.0: - self.set_macro("FrameworkSDKDir", NET_BASE, - "sdkinstallrootv2.0") - else: - raise KeyError("sdkinstallrootv2.0") - except KeyError: - raise DistutilsPlatformError( - """Python was built with Visual Studio 2008; -extensions must be built with a compiler than can generate compatible binaries. -Visual Studio 2008 was not found on this system. If you have Cygwin installed, -you can try compiling with MingW32, by passing "-c mingw32" to setup.py.""") - - if version >= 9.0: - self.set_macro("FrameworkVersion", self.vsbase, "clr version") - self.set_macro("WindowsSdkDir", WINSDK_BASE, "currentinstallfolder") - else: - p = r"Software\Microsoft\NET Framework Setup\Product" - for base in HKEYS: - try: - h = RegOpenKeyEx(base, p) - except RegError: - continue - key = RegEnumKey(h, 0) - d = Reg.get_value(base, r"%s\%s" % (p, key)) - self.macros["$(FrameworkVersion)"] = d["version"] - - def sub(self, s): - for k, v in self.macros.items(): - s = s.replace(k, v) - return s - -def get_build_version(): - """Return the version of MSVC that was used to build Python. - - For Python 2.3 and up, the version number is included in - sys.version. For earlier versions, assume the compiler is MSVC 6. - """ - prefix = "MSC v." - i = sys.version.find(prefix) - if i == -1: - return 6 - i = i + len(prefix) - s, rest = sys.version[i:].split(" ", 1) - majorVersion = int(s[:-2]) - 6 - if majorVersion >= 13: - # v13 was skipped and should be v14 - majorVersion += 1 - minorVersion = int(s[2:3]) / 10.0 - # I don't think paths are affected by minor version in version 6 - if majorVersion == 6: - minorVersion = 0 - if majorVersion >= 6: - return majorVersion + minorVersion - # else we don't know what version of the compiler this is - return None - -def normalize_and_reduce_paths(paths): - """Return a list of normalized paths with duplicates removed. - - The current order of paths is maintained. - """ - # Paths are normalized so things like: /a and /a/ aren't both preserved. - reduced_paths = [] - for p in paths: - np = os.path.normpath(p) - # XXX(nnorwitz): O(n**2), if reduced_paths gets long perhaps use a set. - if np not in reduced_paths: - reduced_paths.append(np) - return reduced_paths - -def removeDuplicates(variable): - """Remove duplicate values of an environment variable. - """ - oldList = variable.split(os.pathsep) - newList = [] - for i in oldList: - if i not in newList: - newList.append(i) - newVariable = os.pathsep.join(newList) - return newVariable - -def find_vcvarsall(version): - """Find the vcvarsall.bat file - - At first it tries to find the productdir of VS 2008 in the registry. If - that fails it falls back to the VS90COMNTOOLS env var. - """ - vsbase = VS_BASE % version - try: - productdir = Reg.get_value(r"%s\Setup\VC" % vsbase, - "productdir") - except KeyError: - log.debug("Unable to find productdir in registry") - productdir = None - - if not productdir or not os.path.isdir(productdir): - toolskey = "VS%0.f0COMNTOOLS" % version - toolsdir = os.environ.get(toolskey, None) - - if toolsdir and os.path.isdir(toolsdir): - productdir = os.path.join(toolsdir, os.pardir, os.pardir, "VC") - productdir = os.path.abspath(productdir) - if not os.path.isdir(productdir): - log.debug("%s is not a valid directory" % productdir) - return None - else: - log.debug("Env var %s is not set or invalid" % toolskey) - if not productdir: - log.debug("No productdir found") - return None - vcvarsall = os.path.join(productdir, "vcvarsall.bat") - if os.path.isfile(vcvarsall): - return vcvarsall - log.debug("Unable to find vcvarsall.bat") - return None - -def query_vcvarsall(version, arch="x86"): - """Launch vcvarsall.bat and read the settings from its environment - """ - vcvarsall = find_vcvarsall(version) - interesting = set(("include", "lib", "libpath", "path")) - result = {} - - if vcvarsall is None: - raise DistutilsPlatformError("Unable to find vcvarsall.bat") - log.debug("Calling 'vcvarsall.bat %s' (version=%s)", arch, version) - popen = subprocess.Popen('"%s" %s & set' % (vcvarsall, arch), - stdout=subprocess.PIPE, - stderr=subprocess.PIPE) - try: - stdout, stderr = popen.communicate() - if popen.wait() != 0: - raise DistutilsPlatformError(stderr.decode("mbcs")) - - stdout = stdout.decode("mbcs") - for line in stdout.split("\n"): - line = Reg.convert_mbcs(line) - if '=' not in line: - continue - line = line.strip() - key, value = line.split('=', 1) - key = key.lower() - if key in interesting: - if value.endswith(os.pathsep): - value = value[:-1] - result[key] = removeDuplicates(value) - - finally: - popen.stdout.close() - popen.stderr.close() - - if len(result) != len(interesting): - raise ValueError(str(list(result.keys()))) - - return result - -# More globals -VERSION = get_build_version() -if VERSION < 8.0: - raise DistutilsPlatformError("VC %0.1f is not supported by this module" % VERSION) -# MACROS = MacroExpander(VERSION) - -class MSVCCompiler(CCompiler) : - """Concrete class that implements an interface to Microsoft Visual C++, - as defined by the CCompiler abstract class.""" - - compiler_type = 'msvc' - - # Just set this so CCompiler's constructor doesn't barf. We currently - # don't use the 'set_executables()' bureaucracy provided by CCompiler, - # as it really isn't necessary for this sort of single-compiler class. - # Would be nice to have a consistent interface with UnixCCompiler, - # though, so it's worth thinking about. - executables = {} - - # Private class data (need to distinguish C from C++ source for compiler) - _c_extensions = ['.c'] - _cpp_extensions = ['.cc', '.cpp', '.cxx'] - _rc_extensions = ['.rc'] - _mc_extensions = ['.mc'] - - # Needed for the filename generation methods provided by the - # base class, CCompiler. - src_extensions = (_c_extensions + _cpp_extensions + - _rc_extensions + _mc_extensions) - res_extension = '.res' - obj_extension = '.obj' - static_lib_extension = '.lib' - shared_lib_extension = '.dll' - static_lib_format = shared_lib_format = '%s%s' - exe_extension = '.exe' - - def __init__(self, verbose=0, dry_run=0, force=0): - CCompiler.__init__ (self, verbose, dry_run, force) - self.__version = VERSION - self.__root = r"Software\Microsoft\VisualStudio" - # self.__macros = MACROS - self.__paths = [] - # target platform (.plat_name is consistent with 'bdist') - self.plat_name = None - self.__arch = None # deprecated name - self.initialized = False - - def initialize(self, plat_name=None): - # multi-init means we would need to check platform same each time... - assert not self.initialized, "don't init multiple times" - if plat_name is None: - plat_name = get_platform() - # sanity check for platforms to prevent obscure errors later. - ok_plats = 'win32', 'win-amd64', 'win-ia64' - if plat_name not in ok_plats: - raise DistutilsPlatformError("--plat-name must be one of %s" % - (ok_plats,)) - - if "DISTUTILS_USE_SDK" in os.environ and "MSSdk" in os.environ and self.find_exe("cl.exe"): - # Assume that the SDK set up everything alright; don't try to be - # smarter - self.cc = "cl.exe" - self.linker = "link.exe" - self.lib = "lib.exe" - self.rc = "rc.exe" - self.mc = "mc.exe" - else: - # On x86, 'vcvars32.bat amd64' creates an env that doesn't work; - # to cross compile, you use 'x86_amd64'. - # On AMD64, 'vcvars32.bat amd64' is a native build env; to cross - # compile use 'x86' (ie, it runs the x86 compiler directly) - # No idea how itanium handles this, if at all. - if plat_name == get_platform() or plat_name == 'win32': - # native build or cross-compile to win32 - plat_spec = PLAT_TO_VCVARS[plat_name] - else: - # cross compile from win32 -> some 64bit - plat_spec = PLAT_TO_VCVARS[get_platform()] + '_' + \ - PLAT_TO_VCVARS[plat_name] - - vc_env = query_vcvarsall(VERSION, plat_spec) - - self.__paths = vc_env['path'].split(os.pathsep) - os.environ['lib'] = vc_env['lib'] - os.environ['include'] = vc_env['include'] - - if len(self.__paths) == 0: - raise DistutilsPlatformError("Python was built with %s, " - "and extensions need to be built with the same " - "version of the compiler, but it isn't installed." - % self.__product) - - self.cc = self.find_exe("cl.exe") - self.linker = self.find_exe("link.exe") - self.lib = self.find_exe("lib.exe") - self.rc = self.find_exe("rc.exe") # resource compiler - self.mc = self.find_exe("mc.exe") # message compiler - #self.set_path_env_var('lib') - #self.set_path_env_var('include') - - # extend the MSVC path with the current path - try: - for p in os.environ['path'].split(';'): - self.__paths.append(p) - except KeyError: - pass - self.__paths = normalize_and_reduce_paths(self.__paths) - os.environ['path'] = ";".join(self.__paths) - - self.preprocess_options = None - if self.__arch == "x86": - self.compile_options = [ '/nologo', '/Ox', '/MD', '/W3', - '/DNDEBUG'] - self.compile_options_debug = ['/nologo', '/Od', '/MDd', '/W3', - '/Z7', '/D_DEBUG'] - else: - # Win64 - self.compile_options = [ '/nologo', '/Ox', '/MD', '/W3', '/GS-' , - '/DNDEBUG'] - self.compile_options_debug = ['/nologo', '/Od', '/MDd', '/W3', '/GS-', - '/Z7', '/D_DEBUG'] - - self.ldflags_shared = ['/DLL', '/nologo', '/INCREMENTAL:NO'] - if self.__version >= 7: - self.ldflags_shared_debug = [ - '/DLL', '/nologo', '/INCREMENTAL:no', '/DEBUG' - ] - self.ldflags_static = [ '/nologo'] - - self.initialized = True - - # -- Worker methods ------------------------------------------------ - - def object_filenames(self, - source_filenames, - strip_dir=0, - output_dir=''): - # Copied from ccompiler.py, extended to return .res as 'object'-file - # for .rc input file - if output_dir is None: output_dir = '' - obj_names = [] - for src_name in source_filenames: - (base, ext) = os.path.splitext (src_name) - base = os.path.splitdrive(base)[1] # Chop off the drive - base = base[os.path.isabs(base):] # If abs, chop off leading / - if ext not in self.src_extensions: - # Better to raise an exception instead of silently continuing - # and later complain about sources and targets having - # different lengths - raise CompileError ("Don't know how to compile %s" % src_name) - if strip_dir: - base = os.path.basename (base) - if ext in self._rc_extensions: - obj_names.append (os.path.join (output_dir, - base + self.res_extension)) - elif ext in self._mc_extensions: - obj_names.append (os.path.join (output_dir, - base + self.res_extension)) - else: - obj_names.append (os.path.join (output_dir, - base + self.obj_extension)) - return obj_names - - - def compile(self, sources, - output_dir=None, macros=None, include_dirs=None, debug=0, - extra_preargs=None, extra_postargs=None, depends=None): - - if not self.initialized: - self.initialize() - compile_info = self._setup_compile(output_dir, macros, include_dirs, - sources, depends, extra_postargs) - macros, objects, extra_postargs, pp_opts, build = compile_info - - compile_opts = extra_preargs or [] - compile_opts.append ('/c') - if debug: - compile_opts.extend(self.compile_options_debug) - else: - compile_opts.extend(self.compile_options) - - for obj in objects: - try: - src, ext = build[obj] - except KeyError: - continue - if debug: - # pass the full pathname to MSVC in debug mode, - # this allows the debugger to find the source file - # without asking the user to browse for it - src = os.path.abspath(src) - - if ext in self._c_extensions: - input_opt = "/Tc" + src - elif ext in self._cpp_extensions: - input_opt = "/Tp" + src - elif ext in self._rc_extensions: - # compile .RC to .RES file - input_opt = src - output_opt = "/fo" + obj - try: - self.spawn([self.rc] + pp_opts + - [output_opt] + [input_opt]) - except DistutilsExecError as msg: - raise CompileError(msg) - continue - elif ext in self._mc_extensions: - # Compile .MC to .RC file to .RES file. - # * '-h dir' specifies the directory for the - # generated include file - # * '-r dir' specifies the target directory of the - # generated RC file and the binary message resource - # it includes - # - # For now (since there are no options to change this), - # we use the source-directory for the include file and - # the build directory for the RC file and message - # resources. This works at least for win32all. - h_dir = os.path.dirname(src) - rc_dir = os.path.dirname(obj) - try: - # first compile .MC to .RC and .H file - self.spawn([self.mc] + - ['-h', h_dir, '-r', rc_dir] + [src]) - base, _ = os.path.splitext (os.path.basename (src)) - rc_file = os.path.join (rc_dir, base + '.rc') - # then compile .RC to .RES file - self.spawn([self.rc] + - ["/fo" + obj] + [rc_file]) - - except DistutilsExecError as msg: - raise CompileError(msg) - continue - else: - # how to handle this file? - raise CompileError("Don't know how to compile %s to %s" - % (src, obj)) - - output_opt = "/Fo" + obj - try: - self.spawn([self.cc] + compile_opts + pp_opts + - [input_opt, output_opt] + - extra_postargs) - except DistutilsExecError as msg: - raise CompileError(msg) - - return objects - - - def create_static_lib(self, - objects, - output_libname, - output_dir=None, - debug=0, - target_lang=None): - - if not self.initialized: - self.initialize() - (objects, output_dir) = self._fix_object_args(objects, output_dir) - output_filename = self.library_filename(output_libname, - output_dir=output_dir) - - if self._need_link(objects, output_filename): - lib_args = objects + ['/OUT:' + output_filename] - if debug: - pass # XXX what goes here? - try: - self.spawn([self.lib] + lib_args) - except DistutilsExecError as msg: - raise LibError(msg) - else: - log.debug("skipping %s (up-to-date)", output_filename) - - - def link(self, - target_desc, - objects, - output_filename, - output_dir=None, - libraries=None, - library_dirs=None, - runtime_library_dirs=None, - export_symbols=None, - debug=0, - extra_preargs=None, - extra_postargs=None, - build_temp=None, - target_lang=None): - - if not self.initialized: - self.initialize() - (objects, output_dir) = self._fix_object_args(objects, output_dir) - fixed_args = self._fix_lib_args(libraries, library_dirs, - runtime_library_dirs) - (libraries, library_dirs, runtime_library_dirs) = fixed_args - - if runtime_library_dirs: - self.warn ("I don't know what to do with 'runtime_library_dirs': " - + str (runtime_library_dirs)) - - lib_opts = gen_lib_options(self, - library_dirs, runtime_library_dirs, - libraries) - if output_dir is not None: - output_filename = os.path.join(output_dir, output_filename) - - if self._need_link(objects, output_filename): - if target_desc == CCompiler.EXECUTABLE: - if debug: - ldflags = self.ldflags_shared_debug[1:] - else: - ldflags = self.ldflags_shared[1:] - else: - if debug: - ldflags = self.ldflags_shared_debug - else: - ldflags = self.ldflags_shared - - export_opts = [] - for sym in (export_symbols or []): - export_opts.append("/EXPORT:" + sym) - - ld_args = (ldflags + lib_opts + export_opts + - objects + ['/OUT:' + output_filename]) - - # The MSVC linker generates .lib and .exp files, which cannot be - # suppressed by any linker switches. The .lib files may even be - # needed! Make sure they are generated in the temporary build - # directory. Since they have different names for debug and release - # builds, they can go into the same directory. - build_temp = os.path.dirname(objects[0]) - if export_symbols is not None: - (dll_name, dll_ext) = os.path.splitext( - os.path.basename(output_filename)) - implib_file = os.path.join( - build_temp, - self.library_filename(dll_name)) - ld_args.append ('/IMPLIB:' + implib_file) - - self.manifest_setup_ldargs(output_filename, build_temp, ld_args) - - if extra_preargs: - ld_args[:0] = extra_preargs - if extra_postargs: - ld_args.extend(extra_postargs) - - self.mkpath(os.path.dirname(output_filename)) - try: - self.spawn([self.linker] + ld_args) - except DistutilsExecError as msg: - raise LinkError(msg) - - # embed the manifest - # XXX - this is somewhat fragile - if mt.exe fails, distutils - # will still consider the DLL up-to-date, but it will not have a - # manifest. Maybe we should link to a temp file? OTOH, that - # implies a build environment error that shouldn't go undetected. - mfinfo = self.manifest_get_embed_info(target_desc, ld_args) - if mfinfo is not None: - mffilename, mfid = mfinfo - out_arg = '-outputresource:%s;%s' % (output_filename, mfid) - try: - self.spawn(['mt.exe', '-nologo', '-manifest', - mffilename, out_arg]) - except DistutilsExecError as msg: - raise LinkError(msg) - else: - log.debug("skipping %s (up-to-date)", output_filename) - - def manifest_setup_ldargs(self, output_filename, build_temp, ld_args): - # If we need a manifest at all, an embedded manifest is recommended. - # See MSDN article titled - # "How to: Embed a Manifest Inside a C/C++ Application" - # (currently at http://msdn2.microsoft.com/en-us/library/ms235591(VS.80).aspx) - # Ask the linker to generate the manifest in the temp dir, so - # we can check it, and possibly embed it, later. - temp_manifest = os.path.join( - build_temp, - os.path.basename(output_filename) + ".manifest") - ld_args.append('/MANIFESTFILE:' + temp_manifest) - - def manifest_get_embed_info(self, target_desc, ld_args): - # If a manifest should be embedded, return a tuple of - # (manifest_filename, resource_id). Returns None if no manifest - # should be embedded. See http://bugs.python.org/issue7833 for why - # we want to avoid any manifest for extension modules if we can) - for arg in ld_args: - if arg.startswith("/MANIFESTFILE:"): - temp_manifest = arg.split(":", 1)[1] - break - else: - # no /MANIFESTFILE so nothing to do. - return None - if target_desc == CCompiler.EXECUTABLE: - # by default, executables always get the manifest with the - # CRT referenced. - mfid = 1 - else: - # Extension modules try and avoid any manifest if possible. - mfid = 2 - temp_manifest = self._remove_visual_c_ref(temp_manifest) - if temp_manifest is None: - return None - return temp_manifest, mfid - - def _remove_visual_c_ref(self, manifest_file): - try: - # Remove references to the Visual C runtime, so they will - # fall through to the Visual C dependency of Python.exe. - # This way, when installed for a restricted user (e.g. - # runtimes are not in WinSxS folder, but in Python's own - # folder), the runtimes do not need to be in every folder - # with .pyd's. - # Returns either the filename of the modified manifest or - # None if no manifest should be embedded. - manifest_f = open(manifest_file) - try: - manifest_buf = manifest_f.read() - finally: - manifest_f.close() - pattern = re.compile( - r"""|)""", - re.DOTALL) - manifest_buf = re.sub(pattern, "", manifest_buf) - pattern = r"\s*" - manifest_buf = re.sub(pattern, "", manifest_buf) - # Now see if any other assemblies are referenced - if not, we - # don't want a manifest embedded. - pattern = re.compile( - r"""|)""", re.DOTALL) - if re.search(pattern, manifest_buf) is None: - return None - - manifest_f = open(manifest_file, 'w') - try: - manifest_f.write(manifest_buf) - return manifest_file - finally: - manifest_f.close() - except OSError: - pass - - # -- Miscellaneous methods ----------------------------------------- - # These are all used by the 'gen_lib_options() function, in - # ccompiler.py. - - def library_dir_option(self, dir): - return "/LIBPATH:" + dir - - def runtime_library_dir_option(self, dir): - raise DistutilsPlatformError( - "don't know how to set runtime library search path for MSVC++") - - def library_option(self, lib): - return self.library_filename(lib) - - - def find_library_file(self, dirs, lib, debug=0): - # Prefer a debugging library if found (and requested), but deal - # with it if we don't have one. - if debug: - try_names = [lib + "_d", lib] - else: - try_names = [lib] - for dir in dirs: - for name in try_names: - libfile = os.path.join(dir, self.library_filename (name)) - if os.path.exists(libfile): - return libfile - else: - # Oops, didn't find it in *any* of 'dirs' - return None - - # Helper methods for using the MSVC registry settings - - def find_exe(self, exe): - """Return path to an MSVC executable program. - - Tries to find the program in several places: first, one of the - MSVC program search paths from the registry; next, the directories - in the PATH environment variable. If any of those work, return an - absolute path that is known to exist. If none of them work, just - return the original program name, 'exe'. - """ - for p in self.__paths: - fn = os.path.join(os.path.abspath(p), exe) - if os.path.isfile(fn): - return fn - - # didn't find it; try existing path - for p in os.environ['Path'].split(';'): - fn = os.path.join(os.path.abspath(p),exe) - if os.path.isfile(fn): - return fn - - return exe diff --git a/Lib/distutils/msvccompiler.py b/Lib/distutils/msvccompiler.py deleted file mode 100644 index 1048cd41593..00000000000 --- a/Lib/distutils/msvccompiler.py +++ /dev/null @@ -1,643 +0,0 @@ -"""distutils.msvccompiler - -Contains MSVCCompiler, an implementation of the abstract CCompiler class -for the Microsoft Visual Studio. -""" - -# Written by Perry Stoll -# hacked by Robin Becker and Thomas Heller to do a better job of -# finding DevStudio (through the registry) - -import sys, os -from distutils.errors import \ - DistutilsExecError, DistutilsPlatformError, \ - CompileError, LibError, LinkError -from distutils.ccompiler import \ - CCompiler, gen_preprocess_options, gen_lib_options -from distutils import log - -_can_read_reg = False -try: - import winreg - - _can_read_reg = True - hkey_mod = winreg - - RegOpenKeyEx = winreg.OpenKeyEx - RegEnumKey = winreg.EnumKey - RegEnumValue = winreg.EnumValue - RegError = winreg.error - -except ImportError: - try: - import win32api - import win32con - _can_read_reg = True - hkey_mod = win32con - - RegOpenKeyEx = win32api.RegOpenKeyEx - RegEnumKey = win32api.RegEnumKey - RegEnumValue = win32api.RegEnumValue - RegError = win32api.error - except ImportError: - log.info("Warning: Can't read registry to find the " - "necessary compiler setting\n" - "Make sure that Python modules winreg, " - "win32api or win32con are installed.") - pass - -if _can_read_reg: - HKEYS = (hkey_mod.HKEY_USERS, - hkey_mod.HKEY_CURRENT_USER, - hkey_mod.HKEY_LOCAL_MACHINE, - hkey_mod.HKEY_CLASSES_ROOT) - -def read_keys(base, key): - """Return list of registry keys.""" - try: - handle = RegOpenKeyEx(base, key) - except RegError: - return None - L = [] - i = 0 - while True: - try: - k = RegEnumKey(handle, i) - except RegError: - break - L.append(k) - i += 1 - return L - -def read_values(base, key): - """Return dict of registry keys and values. - - All names are converted to lowercase. - """ - try: - handle = RegOpenKeyEx(base, key) - except RegError: - return None - d = {} - i = 0 - while True: - try: - name, value, type = RegEnumValue(handle, i) - except RegError: - break - name = name.lower() - d[convert_mbcs(name)] = convert_mbcs(value) - i += 1 - return d - -def convert_mbcs(s): - dec = getattr(s, "decode", None) - if dec is not None: - try: - s = dec("mbcs") - except UnicodeError: - pass - return s - -class MacroExpander: - def __init__(self, version): - self.macros = {} - self.load_macros(version) - - def set_macro(self, macro, path, key): - for base in HKEYS: - d = read_values(base, path) - if d: - self.macros["$(%s)" % macro] = d[key] - break - - def load_macros(self, version): - vsbase = r"Software\Microsoft\VisualStudio\%0.1f" % version - self.set_macro("VCInstallDir", vsbase + r"\Setup\VC", "productdir") - self.set_macro("VSInstallDir", vsbase + r"\Setup\VS", "productdir") - net = r"Software\Microsoft\.NETFramework" - self.set_macro("FrameworkDir", net, "installroot") - try: - if version > 7.0: - self.set_macro("FrameworkSDKDir", net, "sdkinstallrootv1.1") - else: - self.set_macro("FrameworkSDKDir", net, "sdkinstallroot") - except KeyError as exc: # - raise DistutilsPlatformError( - """Python was built with Visual Studio 2003; -extensions must be built with a compiler than can generate compatible binaries. -Visual Studio 2003 was not found on this system. If you have Cygwin installed, -you can try compiling with MingW32, by passing "-c mingw32" to setup.py.""") - - p = r"Software\Microsoft\NET Framework Setup\Product" - for base in HKEYS: - try: - h = RegOpenKeyEx(base, p) - except RegError: - continue - key = RegEnumKey(h, 0) - d = read_values(base, r"%s\%s" % (p, key)) - self.macros["$(FrameworkVersion)"] = d["version"] - - def sub(self, s): - for k, v in self.macros.items(): - s = s.replace(k, v) - return s - -def get_build_version(): - """Return the version of MSVC that was used to build Python. - - For Python 2.3 and up, the version number is included in - sys.version. For earlier versions, assume the compiler is MSVC 6. - """ - prefix = "MSC v." - i = sys.version.find(prefix) - if i == -1: - return 6 - i = i + len(prefix) - s, rest = sys.version[i:].split(" ", 1) - majorVersion = int(s[:-2]) - 6 - if majorVersion >= 13: - # v13 was skipped and should be v14 - majorVersion += 1 - minorVersion = int(s[2:3]) / 10.0 - # I don't think paths are affected by minor version in version 6 - if majorVersion == 6: - minorVersion = 0 - if majorVersion >= 6: - return majorVersion + minorVersion - # else we don't know what version of the compiler this is - return None - -def get_build_architecture(): - """Return the processor architecture. - - Possible results are "Intel", "Itanium", or "AMD64". - """ - - prefix = " bit (" - i = sys.version.find(prefix) - if i == -1: - return "Intel" - j = sys.version.find(")", i) - return sys.version[i+len(prefix):j] - -def normalize_and_reduce_paths(paths): - """Return a list of normalized paths with duplicates removed. - - The current order of paths is maintained. - """ - # Paths are normalized so things like: /a and /a/ aren't both preserved. - reduced_paths = [] - for p in paths: - np = os.path.normpath(p) - # XXX(nnorwitz): O(n**2), if reduced_paths gets long perhaps use a set. - if np not in reduced_paths: - reduced_paths.append(np) - return reduced_paths - - -class MSVCCompiler(CCompiler) : - """Concrete class that implements an interface to Microsoft Visual C++, - as defined by the CCompiler abstract class.""" - - compiler_type = 'msvc' - - # Just set this so CCompiler's constructor doesn't barf. We currently - # don't use the 'set_executables()' bureaucracy provided by CCompiler, - # as it really isn't necessary for this sort of single-compiler class. - # Would be nice to have a consistent interface with UnixCCompiler, - # though, so it's worth thinking about. - executables = {} - - # Private class data (need to distinguish C from C++ source for compiler) - _c_extensions = ['.c'] - _cpp_extensions = ['.cc', '.cpp', '.cxx'] - _rc_extensions = ['.rc'] - _mc_extensions = ['.mc'] - - # Needed for the filename generation methods provided by the - # base class, CCompiler. - src_extensions = (_c_extensions + _cpp_extensions + - _rc_extensions + _mc_extensions) - res_extension = '.res' - obj_extension = '.obj' - static_lib_extension = '.lib' - shared_lib_extension = '.dll' - static_lib_format = shared_lib_format = '%s%s' - exe_extension = '.exe' - - def __init__(self, verbose=0, dry_run=0, force=0): - CCompiler.__init__ (self, verbose, dry_run, force) - self.__version = get_build_version() - self.__arch = get_build_architecture() - if self.__arch == "Intel": - # x86 - if self.__version >= 7: - self.__root = r"Software\Microsoft\VisualStudio" - self.__macros = MacroExpander(self.__version) - else: - self.__root = r"Software\Microsoft\Devstudio" - self.__product = "Visual Studio version %s" % self.__version - else: - # Win64. Assume this was built with the platform SDK - self.__product = "Microsoft SDK compiler %s" % (self.__version + 6) - - self.initialized = False - - def initialize(self): - self.__paths = [] - if "DISTUTILS_USE_SDK" in os.environ and "MSSdk" in os.environ and self.find_exe("cl.exe"): - # Assume that the SDK set up everything alright; don't try to be - # smarter - self.cc = "cl.exe" - self.linker = "link.exe" - self.lib = "lib.exe" - self.rc = "rc.exe" - self.mc = "mc.exe" - else: - self.__paths = self.get_msvc_paths("path") - - if len(self.__paths) == 0: - raise DistutilsPlatformError("Python was built with %s, " - "and extensions need to be built with the same " - "version of the compiler, but it isn't installed." - % self.__product) - - self.cc = self.find_exe("cl.exe") - self.linker = self.find_exe("link.exe") - self.lib = self.find_exe("lib.exe") - self.rc = self.find_exe("rc.exe") # resource compiler - self.mc = self.find_exe("mc.exe") # message compiler - self.set_path_env_var('lib') - self.set_path_env_var('include') - - # extend the MSVC path with the current path - try: - for p in os.environ['path'].split(';'): - self.__paths.append(p) - except KeyError: - pass - self.__paths = normalize_and_reduce_paths(self.__paths) - os.environ['path'] = ";".join(self.__paths) - - self.preprocess_options = None - if self.__arch == "Intel": - self.compile_options = [ '/nologo', '/Ox', '/MD', '/W3', '/GX' , - '/DNDEBUG'] - self.compile_options_debug = ['/nologo', '/Od', '/MDd', '/W3', '/GX', - '/Z7', '/D_DEBUG'] - else: - # Win64 - self.compile_options = [ '/nologo', '/Ox', '/MD', '/W3', '/GS-' , - '/DNDEBUG'] - self.compile_options_debug = ['/nologo', '/Od', '/MDd', '/W3', '/GS-', - '/Z7', '/D_DEBUG'] - - self.ldflags_shared = ['/DLL', '/nologo', '/INCREMENTAL:NO'] - if self.__version >= 7: - self.ldflags_shared_debug = [ - '/DLL', '/nologo', '/INCREMENTAL:no', '/DEBUG' - ] - else: - self.ldflags_shared_debug = [ - '/DLL', '/nologo', '/INCREMENTAL:no', '/pdb:None', '/DEBUG' - ] - self.ldflags_static = [ '/nologo'] - - self.initialized = True - - # -- Worker methods ------------------------------------------------ - - def object_filenames(self, - source_filenames, - strip_dir=0, - output_dir=''): - # Copied from ccompiler.py, extended to return .res as 'object'-file - # for .rc input file - if output_dir is None: output_dir = '' - obj_names = [] - for src_name in source_filenames: - (base, ext) = os.path.splitext (src_name) - base = os.path.splitdrive(base)[1] # Chop off the drive - base = base[os.path.isabs(base):] # If abs, chop off leading / - if ext not in self.src_extensions: - # Better to raise an exception instead of silently continuing - # and later complain about sources and targets having - # different lengths - raise CompileError ("Don't know how to compile %s" % src_name) - if strip_dir: - base = os.path.basename (base) - if ext in self._rc_extensions: - obj_names.append (os.path.join (output_dir, - base + self.res_extension)) - elif ext in self._mc_extensions: - obj_names.append (os.path.join (output_dir, - base + self.res_extension)) - else: - obj_names.append (os.path.join (output_dir, - base + self.obj_extension)) - return obj_names - - - def compile(self, sources, - output_dir=None, macros=None, include_dirs=None, debug=0, - extra_preargs=None, extra_postargs=None, depends=None): - - if not self.initialized: - self.initialize() - compile_info = self._setup_compile(output_dir, macros, include_dirs, - sources, depends, extra_postargs) - macros, objects, extra_postargs, pp_opts, build = compile_info - - compile_opts = extra_preargs or [] - compile_opts.append ('/c') - if debug: - compile_opts.extend(self.compile_options_debug) - else: - compile_opts.extend(self.compile_options) - - for obj in objects: - try: - src, ext = build[obj] - except KeyError: - continue - if debug: - # pass the full pathname to MSVC in debug mode, - # this allows the debugger to find the source file - # without asking the user to browse for it - src = os.path.abspath(src) - - if ext in self._c_extensions: - input_opt = "/Tc" + src - elif ext in self._cpp_extensions: - input_opt = "/Tp" + src - elif ext in self._rc_extensions: - # compile .RC to .RES file - input_opt = src - output_opt = "/fo" + obj - try: - self.spawn([self.rc] + pp_opts + - [output_opt] + [input_opt]) - except DistutilsExecError as msg: - raise CompileError(msg) - continue - elif ext in self._mc_extensions: - # Compile .MC to .RC file to .RES file. - # * '-h dir' specifies the directory for the - # generated include file - # * '-r dir' specifies the target directory of the - # generated RC file and the binary message resource - # it includes - # - # For now (since there are no options to change this), - # we use the source-directory for the include file and - # the build directory for the RC file and message - # resources. This works at least for win32all. - h_dir = os.path.dirname(src) - rc_dir = os.path.dirname(obj) - try: - # first compile .MC to .RC and .H file - self.spawn([self.mc] + - ['-h', h_dir, '-r', rc_dir] + [src]) - base, _ = os.path.splitext (os.path.basename (src)) - rc_file = os.path.join (rc_dir, base + '.rc') - # then compile .RC to .RES file - self.spawn([self.rc] + - ["/fo" + obj] + [rc_file]) - - except DistutilsExecError as msg: - raise CompileError(msg) - continue - else: - # how to handle this file? - raise CompileError("Don't know how to compile %s to %s" - % (src, obj)) - - output_opt = "/Fo" + obj - try: - self.spawn([self.cc] + compile_opts + pp_opts + - [input_opt, output_opt] + - extra_postargs) - except DistutilsExecError as msg: - raise CompileError(msg) - - return objects - - - def create_static_lib(self, - objects, - output_libname, - output_dir=None, - debug=0, - target_lang=None): - - if not self.initialized: - self.initialize() - (objects, output_dir) = self._fix_object_args(objects, output_dir) - output_filename = self.library_filename(output_libname, - output_dir=output_dir) - - if self._need_link(objects, output_filename): - lib_args = objects + ['/OUT:' + output_filename] - if debug: - pass # XXX what goes here? - try: - self.spawn([self.lib] + lib_args) - except DistutilsExecError as msg: - raise LibError(msg) - else: - log.debug("skipping %s (up-to-date)", output_filename) - - - def link(self, - target_desc, - objects, - output_filename, - output_dir=None, - libraries=None, - library_dirs=None, - runtime_library_dirs=None, - export_symbols=None, - debug=0, - extra_preargs=None, - extra_postargs=None, - build_temp=None, - target_lang=None): - - if not self.initialized: - self.initialize() - (objects, output_dir) = self._fix_object_args(objects, output_dir) - fixed_args = self._fix_lib_args(libraries, library_dirs, - runtime_library_dirs) - (libraries, library_dirs, runtime_library_dirs) = fixed_args - - if runtime_library_dirs: - self.warn ("I don't know what to do with 'runtime_library_dirs': " - + str (runtime_library_dirs)) - - lib_opts = gen_lib_options(self, - library_dirs, runtime_library_dirs, - libraries) - if output_dir is not None: - output_filename = os.path.join(output_dir, output_filename) - - if self._need_link(objects, output_filename): - if target_desc == CCompiler.EXECUTABLE: - if debug: - ldflags = self.ldflags_shared_debug[1:] - else: - ldflags = self.ldflags_shared[1:] - else: - if debug: - ldflags = self.ldflags_shared_debug - else: - ldflags = self.ldflags_shared - - export_opts = [] - for sym in (export_symbols or []): - export_opts.append("/EXPORT:" + sym) - - ld_args = (ldflags + lib_opts + export_opts + - objects + ['/OUT:' + output_filename]) - - # The MSVC linker generates .lib and .exp files, which cannot be - # suppressed by any linker switches. The .lib files may even be - # needed! Make sure they are generated in the temporary build - # directory. Since they have different names for debug and release - # builds, they can go into the same directory. - if export_symbols is not None: - (dll_name, dll_ext) = os.path.splitext( - os.path.basename(output_filename)) - implib_file = os.path.join( - os.path.dirname(objects[0]), - self.library_filename(dll_name)) - ld_args.append ('/IMPLIB:' + implib_file) - - if extra_preargs: - ld_args[:0] = extra_preargs - if extra_postargs: - ld_args.extend(extra_postargs) - - self.mkpath(os.path.dirname(output_filename)) - try: - self.spawn([self.linker] + ld_args) - except DistutilsExecError as msg: - raise LinkError(msg) - - else: - log.debug("skipping %s (up-to-date)", output_filename) - - - # -- Miscellaneous methods ----------------------------------------- - # These are all used by the 'gen_lib_options() function, in - # ccompiler.py. - - def library_dir_option(self, dir): - return "/LIBPATH:" + dir - - def runtime_library_dir_option(self, dir): - raise DistutilsPlatformError( - "don't know how to set runtime library search path for MSVC++") - - def library_option(self, lib): - return self.library_filename(lib) - - - def find_library_file(self, dirs, lib, debug=0): - # Prefer a debugging library if found (and requested), but deal - # with it if we don't have one. - if debug: - try_names = [lib + "_d", lib] - else: - try_names = [lib] - for dir in dirs: - for name in try_names: - libfile = os.path.join(dir, self.library_filename (name)) - if os.path.exists(libfile): - return libfile - else: - # Oops, didn't find it in *any* of 'dirs' - return None - - # Helper methods for using the MSVC registry settings - - def find_exe(self, exe): - """Return path to an MSVC executable program. - - Tries to find the program in several places: first, one of the - MSVC program search paths from the registry; next, the directories - in the PATH environment variable. If any of those work, return an - absolute path that is known to exist. If none of them work, just - return the original program name, 'exe'. - """ - for p in self.__paths: - fn = os.path.join(os.path.abspath(p), exe) - if os.path.isfile(fn): - return fn - - # didn't find it; try existing path - for p in os.environ['Path'].split(';'): - fn = os.path.join(os.path.abspath(p),exe) - if os.path.isfile(fn): - return fn - - return exe - - def get_msvc_paths(self, path, platform='x86'): - """Get a list of devstudio directories (include, lib or path). - - Return a list of strings. The list will be empty if unable to - access the registry or appropriate registry keys not found. - """ - if not _can_read_reg: - return [] - - path = path + " dirs" - if self.__version >= 7: - key = (r"%s\%0.1f\VC\VC_OBJECTS_PLATFORM_INFO\Win32\Directories" - % (self.__root, self.__version)) - else: - key = (r"%s\6.0\Build System\Components\Platforms" - r"\Win32 (%s)\Directories" % (self.__root, platform)) - - for base in HKEYS: - d = read_values(base, key) - if d: - if self.__version >= 7: - return self.__macros.sub(d[path]).split(";") - else: - return d[path].split(";") - # MSVC 6 seems to create the registry entries we need only when - # the GUI is run. - if self.__version == 6: - for base in HKEYS: - if read_values(base, r"%s\6.0" % self.__root) is not None: - self.warn("It seems you have Visual Studio 6 installed, " - "but the expected registry settings are not present.\n" - "You must at least run the Visual Studio GUI once " - "so that these entries are created.") - break - return [] - - def set_path_env_var(self, name): - """Set environment variable 'name' to an MSVC path type value. - - This is equivalent to a SET command prior to execution of spawned - commands. - """ - - if name == "lib": - p = self.get_msvc_paths("library") - else: - p = self.get_msvc_paths(name) - if p: - os.environ[name] = ';'.join(p) - - -if get_build_version() >= 8.0: - log.debug("Importing new compiler from distutils.msvc9compiler") - OldMSVCCompiler = MSVCCompiler - from distutils.msvc9compiler import MSVCCompiler - # get_build_architecture not really relevant now we support cross-compile - from distutils.msvc9compiler import MacroExpander diff --git a/Lib/distutils/spawn.py b/Lib/distutils/spawn.py deleted file mode 100644 index 53876880932..00000000000 --- a/Lib/distutils/spawn.py +++ /dev/null @@ -1,192 +0,0 @@ -"""distutils.spawn - -Provides the 'spawn()' function, a front-end to various platform- -specific functions for launching another program in a sub-process. -Also provides the 'find_executable()' to search the path for a given -executable name. -""" - -import sys -import os - -from distutils.errors import DistutilsPlatformError, DistutilsExecError -from distutils.debug import DEBUG -from distutils import log - -def spawn(cmd, search_path=1, verbose=0, dry_run=0): - """Run another program, specified as a command list 'cmd', in a new process. - - 'cmd' is just the argument list for the new process, ie. - cmd[0] is the program to run and cmd[1:] are the rest of its arguments. - There is no way to run a program with a name different from that of its - executable. - - If 'search_path' is true (the default), the system's executable - search path will be used to find the program; otherwise, cmd[0] - must be the exact path to the executable. If 'dry_run' is true, - the command will not actually be run. - - Raise DistutilsExecError if running the program fails in any way; just - return on success. - """ - # cmd is documented as a list, but just in case some code passes a tuple - # in, protect our %-formatting code against horrible death - cmd = list(cmd) - if os.name == 'posix': - _spawn_posix(cmd, search_path, dry_run=dry_run) - elif os.name == 'nt': - _spawn_nt(cmd, search_path, dry_run=dry_run) - else: - raise DistutilsPlatformError( - "don't know how to spawn programs on platform '%s'" % os.name) - -def _nt_quote_args(args): - """Quote command-line arguments for DOS/Windows conventions. - - Just wraps every argument which contains blanks in double quotes, and - returns a new argument list. - """ - # XXX this doesn't seem very robust to me -- but if the Windows guys - # say it'll work, I guess I'll have to accept it. (What if an arg - # contains quotes? What other magic characters, other than spaces, - # have to be escaped? Is there an escaping mechanism other than - # quoting?) - for i, arg in enumerate(args): - if ' ' in arg: - args[i] = '"%s"' % arg - return args - -def _spawn_nt(cmd, search_path=1, verbose=0, dry_run=0): - executable = cmd[0] - cmd = _nt_quote_args(cmd) - if search_path: - # either we find one or it stays the same - executable = find_executable(executable) or executable - log.info(' '.join([executable] + cmd[1:])) - if not dry_run: - # spawn for NT requires a full path to the .exe - try: - rc = os.spawnv(os.P_WAIT, executable, cmd) - except OSError as exc: - # this seems to happen when the command isn't found - if not DEBUG: - cmd = executable - raise DistutilsExecError( - "command %r failed: %s" % (cmd, exc.args[-1])) - if rc != 0: - # and this reflects the command running but failing - if not DEBUG: - cmd = executable - raise DistutilsExecError( - "command %r failed with exit status %d" % (cmd, rc)) - -if sys.platform == 'darwin': - from distutils import sysconfig - _cfg_target = None - _cfg_target_split = None - -def _spawn_posix(cmd, search_path=1, verbose=0, dry_run=0): - log.info(' '.join(cmd)) - if dry_run: - return - executable = cmd[0] - exec_fn = search_path and os.execvp or os.execv - env = None - if sys.platform == 'darwin': - global _cfg_target, _cfg_target_split - if _cfg_target is None: - _cfg_target = sysconfig.get_config_var( - 'MACOSX_DEPLOYMENT_TARGET') or '' - if _cfg_target: - _cfg_target_split = [int(x) for x in _cfg_target.split('.')] - if _cfg_target: - # ensure that the deployment target of build process is not less - # than that used when the interpreter was built. This ensures - # extension modules are built with correct compatibility values - cur_target = os.environ.get('MACOSX_DEPLOYMENT_TARGET', _cfg_target) - if _cfg_target_split > [int(x) for x in cur_target.split('.')]: - my_msg = ('$MACOSX_DEPLOYMENT_TARGET mismatch: ' - 'now "%s" but "%s" during configure' - % (cur_target, _cfg_target)) - raise DistutilsPlatformError(my_msg) - env = dict(os.environ, - MACOSX_DEPLOYMENT_TARGET=cur_target) - exec_fn = search_path and os.execvpe or os.execve - pid = os.fork() - if pid == 0: # in the child - try: - if env is None: - exec_fn(executable, cmd) - else: - exec_fn(executable, cmd, env) - except OSError as e: - if not DEBUG: - cmd = executable - sys.stderr.write("unable to execute %r: %s\n" - % (cmd, e.strerror)) - os._exit(1) - - if not DEBUG: - cmd = executable - sys.stderr.write("unable to execute %r for unknown reasons" % cmd) - os._exit(1) - else: # in the parent - # Loop until the child either exits or is terminated by a signal - # (ie. keep waiting if it's merely stopped) - while True: - try: - pid, status = os.waitpid(pid, 0) - except OSError as exc: - if not DEBUG: - cmd = executable - raise DistutilsExecError( - "command %r failed: %s" % (cmd, exc.args[-1])) - if os.WIFSIGNALED(status): - if not DEBUG: - cmd = executable - raise DistutilsExecError( - "command %r terminated by signal %d" - % (cmd, os.WTERMSIG(status))) - elif os.WIFEXITED(status): - exit_status = os.WEXITSTATUS(status) - if exit_status == 0: - return # hey, it succeeded! - else: - if not DEBUG: - cmd = executable - raise DistutilsExecError( - "command %r failed with exit status %d" - % (cmd, exit_status)) - elif os.WIFSTOPPED(status): - continue - else: - if not DEBUG: - cmd = executable - raise DistutilsExecError( - "unknown error executing %r: termination status %d" - % (cmd, status)) - -def find_executable(executable, path=None): - """Tries to find 'executable' in the directories listed in 'path'. - - A string listing directories separated by 'os.pathsep'; defaults to - os.environ['PATH']. Returns the complete filename or None if not found. - """ - if path is None: - path = os.environ.get('PATH', os.defpath) - - paths = path.split(os.pathsep) - base, ext = os.path.splitext(executable) - - if (sys.platform == 'win32') and (ext != '.exe'): - executable = executable + '.exe' - - if not os.path.isfile(executable): - for p in paths: - f = os.path.join(p, executable) - if os.path.isfile(f): - # the file exists, we have a shot at spawn working - return f - return None - else: - return executable diff --git a/Lib/distutils/sysconfig.py b/Lib/distutils/sysconfig.py deleted file mode 100644 index 3a5984f5c01..00000000000 --- a/Lib/distutils/sysconfig.py +++ /dev/null @@ -1,556 +0,0 @@ -"""Provide access to Python's configuration information. The specific -configuration variables available depend heavily on the platform and -configuration. The values may be retrieved using -get_config_var(name), and the list of variables is available via -get_config_vars().keys(). Additional convenience functions are also -available. - -Written by: Fred L. Drake, Jr. -Email: -""" - -import _imp -import os -import re -import sys - -from .errors import DistutilsPlatformError - -# These are needed in a couple of spots, so just compute them once. -PREFIX = os.path.normpath(sys.prefix) -EXEC_PREFIX = os.path.normpath(sys.exec_prefix) -BASE_PREFIX = os.path.normpath(sys.base_prefix) -BASE_EXEC_PREFIX = os.path.normpath(sys.base_exec_prefix) - -# Path to the base directory of the project. On Windows the binary may -# live in project/PCbuild/win32 or project/PCbuild/amd64. -# set for cross builds -if "_PYTHON_PROJECT_BASE" in os.environ: - project_base = os.path.abspath(os.environ["_PYTHON_PROJECT_BASE"]) -else: - if sys.executable: - project_base = os.path.dirname(os.path.abspath(sys.executable)) - else: - # sys.executable can be empty if argv[0] has been changed and Python is - # unable to retrieve the real program name - project_base = os.getcwd() - - -# python_build: (Boolean) if true, we're either building Python or -# building an extension with an un-installed Python, so we use -# different (hard-wired) directories. -def _is_python_source_dir(d): - for fn in ("Setup", "Setup.local"): - if os.path.isfile(os.path.join(d, "Modules", fn)): - return True - return False - -_sys_home = getattr(sys, '_home', None) - -if os.name == 'nt': - def _fix_pcbuild(d): - if d and os.path.normcase(d).startswith( - os.path.normcase(os.path.join(PREFIX, "PCbuild"))): - return PREFIX - return d - project_base = _fix_pcbuild(project_base) - _sys_home = _fix_pcbuild(_sys_home) - -def _python_build(): - if _sys_home: - return _is_python_source_dir(_sys_home) - return _is_python_source_dir(project_base) - -python_build = _python_build() - - -# Calculate the build qualifier flags if they are defined. Adding the flags -# to the include and lib directories only makes sense for an installation, not -# an in-source build. -build_flags = '' -try: - if not python_build: - build_flags = sys.abiflags -except AttributeError: - # It's not a configure-based build, so the sys module doesn't have - # this attribute, which is fine. - pass - -def get_python_version(): - """Return a string containing the major and minor Python version, - leaving off the patchlevel. Sample return values could be '1.5' - or '2.2'. - """ - return '%d.%d' % sys.version_info[:2] - - -def get_python_inc(plat_specific=0, prefix=None): - """Return the directory containing installed Python header files. - - If 'plat_specific' is false (the default), this is the path to the - non-platform-specific header files, i.e. Python.h and so on; - otherwise, this is the path to platform-specific header files - (namely pyconfig.h). - - If 'prefix' is supplied, use it instead of sys.base_prefix or - sys.base_exec_prefix -- i.e., ignore 'plat_specific'. - """ - if prefix is None: - prefix = plat_specific and BASE_EXEC_PREFIX or BASE_PREFIX - if os.name == "posix": - if python_build: - # Assume the executable is in the build directory. The - # pyconfig.h file should be in the same directory. Since - # the build directory may not be the source directory, we - # must use "srcdir" from the makefile to find the "Include" - # directory. - if plat_specific: - return _sys_home or project_base - else: - incdir = os.path.join(get_config_var('srcdir'), 'Include') - return os.path.normpath(incdir) - python_dir = 'python' + get_python_version() + build_flags - return os.path.join(prefix, "include", python_dir) - elif os.name == "nt": - if python_build: - # Include both the include and PC dir to ensure we can find - # pyconfig.h - return (os.path.join(prefix, "include") + os.path.pathsep + - os.path.join(prefix, "PC")) - return os.path.join(prefix, "include") - else: - raise DistutilsPlatformError( - "I don't know where Python installs its C header files " - "on platform '%s'" % os.name) - - -def get_python_lib(plat_specific=0, standard_lib=0, prefix=None): - """Return the directory containing the Python library (standard or - site additions). - - If 'plat_specific' is true, return the directory containing - platform-specific modules, i.e. any module from a non-pure-Python - module distribution; otherwise, return the platform-shared library - directory. If 'standard_lib' is true, return the directory - containing standard Python library modules; otherwise, return the - directory for site-specific modules. - - If 'prefix' is supplied, use it instead of sys.base_prefix or - sys.base_exec_prefix -- i.e., ignore 'plat_specific'. - """ - if prefix is None: - if standard_lib: - prefix = plat_specific and BASE_EXEC_PREFIX or BASE_PREFIX - else: - prefix = plat_specific and EXEC_PREFIX or PREFIX - - if os.name == "posix": - if plat_specific or standard_lib: - # Platform-specific modules (any module from a non-pure-Python - # module distribution) or standard Python library modules. - libdir = sys.platlibdir - else: - # Pure Python - libdir = "lib" - libpython = os.path.join(prefix, libdir, - # XXX RUSTPYTHON: changed from python->rustpython - "rustpython" + get_python_version()) - if standard_lib: - return libpython - else: - return os.path.join(libpython, "site-packages") - elif os.name == "nt": - if standard_lib: - return os.path.join(prefix, "Lib") - else: - return os.path.join(prefix, "Lib", "site-packages") - else: - raise DistutilsPlatformError( - "I don't know where Python installs its library " - "on platform '%s'" % os.name) - - - -def customize_compiler(compiler): - """Do any platform-specific customization of a CCompiler instance. - - Mainly needed on Unix, so we can plug in the information that - varies across Unices and is stored in Python's Makefile. - """ - if compiler.compiler_type == "unix": - if sys.platform == "darwin": - # Perform first-time customization of compiler-related - # config vars on OS X now that we know we need a compiler. - # This is primarily to support Pythons from binary - # installers. The kind and paths to build tools on - # the user system may vary significantly from the system - # that Python itself was built on. Also the user OS - # version and build tools may not support the same set - # of CPU architectures for universal builds. - global _config_vars - # Use get_config_var() to ensure _config_vars is initialized. - if not get_config_var('CUSTOMIZED_OSX_COMPILER'): - import _osx_support - _osx_support.customize_compiler(_config_vars) - _config_vars['CUSTOMIZED_OSX_COMPILER'] = 'True' - - (cc, cxx, cflags, ccshared, ldshared, shlib_suffix, ar, ar_flags) = \ - get_config_vars('CC', 'CXX', 'CFLAGS', - 'CCSHARED', 'LDSHARED', 'SHLIB_SUFFIX', 'AR', 'ARFLAGS') - - if 'CC' in os.environ: - newcc = os.environ['CC'] - if (sys.platform == 'darwin' - and 'LDSHARED' not in os.environ - and ldshared.startswith(cc)): - # On OS X, if CC is overridden, use that as the default - # command for LDSHARED as well - ldshared = newcc + ldshared[len(cc):] - cc = newcc - if 'CXX' in os.environ: - cxx = os.environ['CXX'] - if 'LDSHARED' in os.environ: - ldshared = os.environ['LDSHARED'] - if 'CPP' in os.environ: - cpp = os.environ['CPP'] - else: - cpp = cc + " -E" # not always - if 'LDFLAGS' in os.environ: - ldshared = ldshared + ' ' + os.environ['LDFLAGS'] - if 'CFLAGS' in os.environ: - cflags = cflags + ' ' + os.environ['CFLAGS'] - ldshared = ldshared + ' ' + os.environ['CFLAGS'] - if 'CPPFLAGS' in os.environ: - cpp = cpp + ' ' + os.environ['CPPFLAGS'] - cflags = cflags + ' ' + os.environ['CPPFLAGS'] - ldshared = ldshared + ' ' + os.environ['CPPFLAGS'] - if 'AR' in os.environ: - ar = os.environ['AR'] - if 'ARFLAGS' in os.environ: - archiver = ar + ' ' + os.environ['ARFLAGS'] - else: - archiver = ar + ' ' + ar_flags - - cc_cmd = cc + ' ' + cflags - compiler.set_executables( - preprocessor=cpp, - compiler=cc_cmd, - compiler_so=cc_cmd + ' ' + ccshared, - compiler_cxx=cxx, - linker_so=ldshared, - linker_exe=cc, - archiver=archiver) - - compiler.shared_lib_extension = shlib_suffix - - -def get_config_h_filename(): - """Return full pathname of installed pyconfig.h file.""" - if python_build: - if os.name == "nt": - inc_dir = os.path.join(_sys_home or project_base, "PC") - else: - inc_dir = _sys_home or project_base - else: - inc_dir = get_python_inc(plat_specific=1) - - return os.path.join(inc_dir, 'pyconfig.h') - - -def get_makefile_filename(): - """Return full pathname of installed Makefile from the Python build.""" - if python_build: - return os.path.join(_sys_home or project_base, "Makefile") - lib_dir = get_python_lib(plat_specific=0, standard_lib=1) - config_file = 'config-{}{}'.format(get_python_version(), build_flags) - if hasattr(sys.implementation, '_multiarch'): - config_file += '-%s' % sys.implementation._multiarch - return os.path.join(lib_dir, config_file, 'Makefile') - - -def parse_config_h(fp, g=None): - """Parse a config.h-style file. - - A dictionary containing name/value pairs is returned. If an - optional dictionary is passed in as the second argument, it is - used instead of a new dictionary. - """ - if g is None: - g = {} - define_rx = re.compile("#define ([A-Z][A-Za-z0-9_]+) (.*)\n") - undef_rx = re.compile("/[*] #undef ([A-Z][A-Za-z0-9_]+) [*]/\n") - # - while True: - line = fp.readline() - if not line: - break - m = define_rx.match(line) - if m: - n, v = m.group(1, 2) - try: v = int(v) - except ValueError: pass - g[n] = v - else: - m = undef_rx.match(line) - if m: - g[m.group(1)] = 0 - return g - - -# Regexes needed for parsing Makefile (and similar syntaxes, -# like old-style Setup files). -_variable_rx = re.compile(r"([a-zA-Z][a-zA-Z0-9_]+)\s*=\s*(.*)") -_findvar1_rx = re.compile(r"\$\(([A-Za-z][A-Za-z0-9_]*)\)") -_findvar2_rx = re.compile(r"\${([A-Za-z][A-Za-z0-9_]*)}") - -def parse_makefile(fn, g=None): - """Parse a Makefile-style file. - - A dictionary containing name/value pairs is returned. If an - optional dictionary is passed in as the second argument, it is - used instead of a new dictionary. - """ - from distutils.text_file import TextFile - fp = TextFile(fn, strip_comments=1, skip_blanks=1, join_lines=1, errors="surrogateescape") - - if g is None: - g = {} - done = {} - notdone = {} - - while True: - line = fp.readline() - if line is None: # eof - break - m = _variable_rx.match(line) - if m: - n, v = m.group(1, 2) - v = v.strip() - # `$$' is a literal `$' in make - tmpv = v.replace('$$', '') - - if "$" in tmpv: - notdone[n] = v - else: - try: - v = int(v) - except ValueError: - # insert literal `$' - done[n] = v.replace('$$', '$') - else: - done[n] = v - - # Variables with a 'PY_' prefix in the makefile. These need to - # be made available without that prefix through sysconfig. - # Special care is needed to ensure that variable expansion works, even - # if the expansion uses the name without a prefix. - renamed_variables = ('CFLAGS', 'LDFLAGS', 'CPPFLAGS') - - # do variable interpolation here - while notdone: - for name in list(notdone): - value = notdone[name] - m = _findvar1_rx.search(value) or _findvar2_rx.search(value) - if m: - n = m.group(1) - found = True - if n in done: - item = str(done[n]) - elif n in notdone: - # get it on a subsequent round - found = False - elif n in os.environ: - # do it like make: fall back to environment - item = os.environ[n] - - elif n in renamed_variables: - if name.startswith('PY_') and name[3:] in renamed_variables: - item = "" - - elif 'PY_' + n in notdone: - found = False - - else: - item = str(done['PY_' + n]) - else: - done[n] = item = "" - if found: - after = value[m.end():] - value = value[:m.start()] + item + after - if "$" in after: - notdone[name] = value - else: - try: value = int(value) - except ValueError: - done[name] = value.strip() - else: - done[name] = value - del notdone[name] - - if name.startswith('PY_') \ - and name[3:] in renamed_variables: - - name = name[3:] - if name not in done: - done[name] = value - else: - # bogus variable reference; just drop it since we can't deal - del notdone[name] - - fp.close() - - # strip spurious spaces - for k, v in done.items(): - if isinstance(v, str): - done[k] = v.strip() - - # save the results in the global dictionary - g.update(done) - return g - - -def expand_makefile_vars(s, vars): - """Expand Makefile-style variables -- "${foo}" or "$(foo)" -- in - 'string' according to 'vars' (a dictionary mapping variable names to - values). Variables not present in 'vars' are silently expanded to the - empty string. The variable values in 'vars' should not contain further - variable expansions; if 'vars' is the output of 'parse_makefile()', - you're fine. Returns a variable-expanded version of 's'. - """ - - # This algorithm does multiple expansion, so if vars['foo'] contains - # "${bar}", it will expand ${foo} to ${bar}, and then expand - # ${bar}... and so forth. This is fine as long as 'vars' comes from - # 'parse_makefile()', which takes care of such expansions eagerly, - # according to make's variable expansion semantics. - - while True: - m = _findvar1_rx.search(s) or _findvar2_rx.search(s) - if m: - (beg, end) = m.span() - s = s[0:beg] + vars.get(m.group(1)) + s[end:] - else: - break - return s - - -_config_vars = None - -def _init_posix(): - """Initialize the module as appropriate for POSIX systems.""" - # _sysconfigdata is generated at build time, see the sysconfig module - name = os.environ.get('_PYTHON_SYSCONFIGDATA_NAME', - '_sysconfigdata_{abi}_{platform}_{multiarch}'.format( - abi=sys.abiflags, - platform=sys.platform, - multiarch=getattr(sys.implementation, '_multiarch', ''), - )) - _temp = __import__(name, globals(), locals(), ['build_time_vars'], 0) - build_time_vars = _temp.build_time_vars - global _config_vars - _config_vars = {} - _config_vars.update(build_time_vars) - - -def _init_nt(): - """Initialize the module as appropriate for NT""" - g = {} - # set basic install directories - g['LIBDEST'] = get_python_lib(plat_specific=0, standard_lib=1) - g['BINLIBDEST'] = get_python_lib(plat_specific=1, standard_lib=1) - - # XXX hmmm.. a normal install puts include files here - g['INCLUDEPY'] = get_python_inc(plat_specific=0) - - g['EXT_SUFFIX'] = _imp.extension_suffixes()[0] - g['EXE'] = ".exe" - g['VERSION'] = get_python_version().replace(".", "") - g['BINDIR'] = os.path.dirname(os.path.abspath(sys.executable)) - - global _config_vars - _config_vars = g - - -def get_config_vars(*args): - """With no arguments, return a dictionary of all configuration - variables relevant for the current platform. Generally this includes - everything needed to build extensions and install both pure modules and - extensions. On Unix, this means every variable defined in Python's - installed Makefile; on Windows it's a much smaller set. - - With arguments, return a list of values that result from looking up - each argument in the configuration variable dictionary. - """ - global _config_vars - if _config_vars is None: - func = globals().get("_init_" + os.name) - if func: - func() - else: - _config_vars = {} - - # Normalized versions of prefix and exec_prefix are handy to have; - # in fact, these are the standard versions used most places in the - # Distutils. - _config_vars['prefix'] = PREFIX - _config_vars['exec_prefix'] = EXEC_PREFIX - - # For backward compatibility, see issue19555 - SO = _config_vars.get('EXT_SUFFIX') - if SO is not None: - _config_vars['SO'] = SO - - # Always convert srcdir to an absolute path - srcdir = _config_vars.get('srcdir', project_base) - if os.name == 'posix': - if python_build: - # If srcdir is a relative path (typically '.' or '..') - # then it should be interpreted relative to the directory - # containing Makefile. - base = os.path.dirname(get_makefile_filename()) - srcdir = os.path.join(base, srcdir) - else: - # srcdir is not meaningful since the installation is - # spread about the filesystem. We choose the - # directory containing the Makefile since we know it - # exists. - srcdir = os.path.dirname(get_makefile_filename()) - _config_vars['srcdir'] = os.path.abspath(os.path.normpath(srcdir)) - - # Convert srcdir into an absolute path if it appears necessary. - # Normally it is relative to the build directory. However, during - # testing, for example, we might be running a non-installed python - # from a different directory. - if python_build and os.name == "posix": - base = project_base - if (not os.path.isabs(_config_vars['srcdir']) and - base != os.getcwd()): - # srcdir is relative and we are not in the same directory - # as the executable. Assume executable is in the build - # directory and make srcdir absolute. - srcdir = os.path.join(base, _config_vars['srcdir']) - _config_vars['srcdir'] = os.path.normpath(srcdir) - - # OS X platforms require special customization to handle - # multi-architecture, multi-os-version installers - if sys.platform == 'darwin': - import _osx_support - _osx_support.customize_config_vars(_config_vars) - - if args: - vals = [] - for name in args: - vals.append(_config_vars.get(name)) - return vals - else: - return _config_vars - -def get_config_var(name): - """Return the value of a single variable using the dictionary - returned by 'get_config_vars()'. Equivalent to - get_config_vars().get(name) - """ - if name == 'SO': - import warnings - warnings.warn('SO is deprecated, use EXT_SUFFIX', DeprecationWarning, 2) - return get_config_vars().get(name) diff --git a/Lib/distutils/text_file.py b/Lib/distutils/text_file.py deleted file mode 100644 index 93abad38f43..00000000000 --- a/Lib/distutils/text_file.py +++ /dev/null @@ -1,286 +0,0 @@ -"""text_file - -provides the TextFile class, which gives an interface to text files -that (optionally) takes care of stripping comments, ignoring blank -lines, and joining lines with backslashes.""" - -import sys, io - - -class TextFile: - """Provides a file-like object that takes care of all the things you - commonly want to do when processing a text file that has some - line-by-line syntax: strip comments (as long as "#" is your - comment character), skip blank lines, join adjacent lines by - escaping the newline (ie. backslash at end of line), strip - leading and/or trailing whitespace. All of these are optional - and independently controllable. - - Provides a 'warn()' method so you can generate warning messages that - report physical line number, even if the logical line in question - spans multiple physical lines. Also provides 'unreadline()' for - implementing line-at-a-time lookahead. - - Constructor is called as: - - TextFile (filename=None, file=None, **options) - - It bombs (RuntimeError) if both 'filename' and 'file' are None; - 'filename' should be a string, and 'file' a file object (or - something that provides 'readline()' and 'close()' methods). It is - recommended that you supply at least 'filename', so that TextFile - can include it in warning messages. If 'file' is not supplied, - TextFile creates its own using 'io.open()'. - - The options are all boolean, and affect the value returned by - 'readline()': - strip_comments [default: true] - strip from "#" to end-of-line, as well as any whitespace - leading up to the "#" -- unless it is escaped by a backslash - lstrip_ws [default: false] - strip leading whitespace from each line before returning it - rstrip_ws [default: true] - strip trailing whitespace (including line terminator!) from - each line before returning it - skip_blanks [default: true} - skip lines that are empty *after* stripping comments and - whitespace. (If both lstrip_ws and rstrip_ws are false, - then some lines may consist of solely whitespace: these will - *not* be skipped, even if 'skip_blanks' is true.) - join_lines [default: false] - if a backslash is the last non-newline character on a line - after stripping comments and whitespace, join the following line - to it to form one "logical line"; if N consecutive lines end - with a backslash, then N+1 physical lines will be joined to - form one logical line. - collapse_join [default: false] - strip leading whitespace from lines that are joined to their - predecessor; only matters if (join_lines and not lstrip_ws) - errors [default: 'strict'] - error handler used to decode the file content - - Note that since 'rstrip_ws' can strip the trailing newline, the - semantics of 'readline()' must differ from those of the builtin file - object's 'readline()' method! In particular, 'readline()' returns - None for end-of-file: an empty string might just be a blank line (or - an all-whitespace line), if 'rstrip_ws' is true but 'skip_blanks' is - not.""" - - default_options = { 'strip_comments': 1, - 'skip_blanks': 1, - 'lstrip_ws': 0, - 'rstrip_ws': 1, - 'join_lines': 0, - 'collapse_join': 0, - 'errors': 'strict', - } - - def __init__(self, filename=None, file=None, **options): - """Construct a new TextFile object. At least one of 'filename' - (a string) and 'file' (a file-like object) must be supplied. - They keyword argument options are described above and affect - the values returned by 'readline()'.""" - if filename is None and file is None: - raise RuntimeError("you must supply either or both of 'filename' and 'file'") - - # set values for all options -- either from client option hash - # or fallback to default_options - for opt in self.default_options.keys(): - if opt in options: - setattr(self, opt, options[opt]) - else: - setattr(self, opt, self.default_options[opt]) - - # sanity check client option hash - for opt in options.keys(): - if opt not in self.default_options: - raise KeyError("invalid TextFile option '%s'" % opt) - - if file is None: - self.open(filename) - else: - self.filename = filename - self.file = file - self.current_line = 0 # assuming that file is at BOF! - - # 'linebuf' is a stack of lines that will be emptied before we - # actually read from the file; it's only populated by an - # 'unreadline()' operation - self.linebuf = [] - - def open(self, filename): - """Open a new file named 'filename'. This overrides both the - 'filename' and 'file' arguments to the constructor.""" - self.filename = filename - self.file = io.open(self.filename, 'r', errors=self.errors) - self.current_line = 0 - - def close(self): - """Close the current file and forget everything we know about it - (filename, current line number).""" - file = self.file - self.file = None - self.filename = None - self.current_line = None - file.close() - - def gen_error(self, msg, line=None): - outmsg = [] - if line is None: - line = self.current_line - outmsg.append(self.filename + ", ") - if isinstance(line, (list, tuple)): - outmsg.append("lines %d-%d: " % tuple(line)) - else: - outmsg.append("line %d: " % line) - outmsg.append(str(msg)) - return "".join(outmsg) - - def error(self, msg, line=None): - raise ValueError("error: " + self.gen_error(msg, line)) - - def warn(self, msg, line=None): - """Print (to stderr) a warning message tied to the current logical - line in the current file. If the current logical line in the - file spans multiple physical lines, the warning refers to the - whole range, eg. "lines 3-5". If 'line' supplied, it overrides - the current line number; it may be a list or tuple to indicate a - range of physical lines, or an integer for a single physical - line.""" - sys.stderr.write("warning: " + self.gen_error(msg, line) + "\n") - - def readline(self): - """Read and return a single logical line from the current file (or - from an internal buffer if lines have previously been "unread" - with 'unreadline()'). If the 'join_lines' option is true, this - may involve reading multiple physical lines concatenated into a - single string. Updates the current line number, so calling - 'warn()' after 'readline()' emits a warning about the physical - line(s) just read. Returns None on end-of-file, since the empty - string can occur if 'rstrip_ws' is true but 'strip_blanks' is - not.""" - # If any "unread" lines waiting in 'linebuf', return the top - # one. (We don't actually buffer read-ahead data -- lines only - # get put in 'linebuf' if the client explicitly does an - # 'unreadline()'. - if self.linebuf: - line = self.linebuf[-1] - del self.linebuf[-1] - return line - - buildup_line = '' - - while True: - # read the line, make it None if EOF - line = self.file.readline() - if line == '': - line = None - - if self.strip_comments and line: - - # Look for the first "#" in the line. If none, never - # mind. If we find one and it's the first character, or - # is not preceded by "\", then it starts a comment -- - # strip the comment, strip whitespace before it, and - # carry on. Otherwise, it's just an escaped "#", so - # unescape it (and any other escaped "#"'s that might be - # lurking in there) and otherwise leave the line alone. - - pos = line.find("#") - if pos == -1: # no "#" -- no comments - pass - - # It's definitely a comment -- either "#" is the first - # character, or it's elsewhere and unescaped. - elif pos == 0 or line[pos-1] != "\\": - # Have to preserve the trailing newline, because it's - # the job of a later step (rstrip_ws) to remove it -- - # and if rstrip_ws is false, we'd better preserve it! - # (NB. this means that if the final line is all comment - # and has no trailing newline, we will think that it's - # EOF; I think that's OK.) - eol = (line[-1] == '\n') and '\n' or '' - line = line[0:pos] + eol - - # If all that's left is whitespace, then skip line - # *now*, before we try to join it to 'buildup_line' -- - # that way constructs like - # hello \\ - # # comment that should be ignored - # there - # result in "hello there". - if line.strip() == "": - continue - else: # it's an escaped "#" - line = line.replace("\\#", "#") - - # did previous line end with a backslash? then accumulate - if self.join_lines and buildup_line: - # oops: end of file - if line is None: - self.warn("continuation line immediately precedes " - "end-of-file") - return buildup_line - - if self.collapse_join: - line = line.lstrip() - line = buildup_line + line - - # careful: pay attention to line number when incrementing it - if isinstance(self.current_line, list): - self.current_line[1] = self.current_line[1] + 1 - else: - self.current_line = [self.current_line, - self.current_line + 1] - # just an ordinary line, read it as usual - else: - if line is None: # eof - return None - - # still have to be careful about incrementing the line number! - if isinstance(self.current_line, list): - self.current_line = self.current_line[1] + 1 - else: - self.current_line = self.current_line + 1 - - # strip whitespace however the client wants (leading and - # trailing, or one or the other, or neither) - if self.lstrip_ws and self.rstrip_ws: - line = line.strip() - elif self.lstrip_ws: - line = line.lstrip() - elif self.rstrip_ws: - line = line.rstrip() - - # blank line (whether we rstrip'ed or not)? skip to next line - # if appropriate - if (line == '' or line == '\n') and self.skip_blanks: - continue - - if self.join_lines: - if line[-1] == '\\': - buildup_line = line[:-1] - continue - - if line[-2:] == '\\\n': - buildup_line = line[0:-2] + '\n' - continue - - # well, I guess there's some actual content there: return it - return line - - def readlines(self): - """Read and return the list of all logical lines remaining in the - current file.""" - lines = [] - while True: - line = self.readline() - if line is None: - return lines - lines.append(line) - - def unreadline(self, line): - """Push 'line' (a string) onto an internal buffer that will be - checked by future 'readline()' calls. Handy for implementing - a parser with line-at-a-time lookahead.""" - self.linebuf.append(line) diff --git a/Lib/distutils/unixccompiler.py b/Lib/distutils/unixccompiler.py deleted file mode 100644 index 4524417e668..00000000000 --- a/Lib/distutils/unixccompiler.py +++ /dev/null @@ -1,333 +0,0 @@ -"""distutils.unixccompiler - -Contains the UnixCCompiler class, a subclass of CCompiler that handles -the "typical" Unix-style command-line C compiler: - * macros defined with -Dname[=value] - * macros undefined with -Uname - * include search directories specified with -Idir - * libraries specified with -lllib - * library search directories specified with -Ldir - * compile handled by 'cc' (or similar) executable with -c option: - compiles .c to .o - * link static library handled by 'ar' command (possibly with 'ranlib') - * link shared library handled by 'cc -shared' -""" - -import os, sys, re - -from distutils import sysconfig -from distutils.dep_util import newer -from distutils.ccompiler import \ - CCompiler, gen_preprocess_options, gen_lib_options -from distutils.errors import \ - DistutilsExecError, CompileError, LibError, LinkError -from distutils import log - -if sys.platform == 'darwin': - import _osx_support - -# XXX Things not currently handled: -# * optimization/debug/warning flags; we just use whatever's in Python's -# Makefile and live with it. Is this adequate? If not, we might -# have to have a bunch of subclasses GNUCCompiler, SGICCompiler, -# SunCCompiler, and I suspect down that road lies madness. -# * even if we don't know a warning flag from an optimization flag, -# we need some way for outsiders to feed preprocessor/compiler/linker -# flags in to us -- eg. a sysadmin might want to mandate certain flags -# via a site config file, or a user might want to set something for -# compiling this module distribution only via the setup.py command -# line, whatever. As long as these options come from something on the -# current system, they can be as system-dependent as they like, and we -# should just happily stuff them into the preprocessor/compiler/linker -# options and carry on. - - -class UnixCCompiler(CCompiler): - - compiler_type = 'unix' - - # These are used by CCompiler in two places: the constructor sets - # instance attributes 'preprocessor', 'compiler', etc. from them, and - # 'set_executable()' allows any of these to be set. The defaults here - # are pretty generic; they will probably have to be set by an outsider - # (eg. using information discovered by the sysconfig about building - # Python extensions). - executables = {'preprocessor' : None, - 'compiler' : ["cc"], - 'compiler_so' : ["cc"], - 'compiler_cxx' : ["cc"], - 'linker_so' : ["cc", "-shared"], - 'linker_exe' : ["cc"], - 'archiver' : ["ar", "-cr"], - 'ranlib' : None, - } - - if sys.platform[:6] == "darwin": - executables['ranlib'] = ["ranlib"] - - # Needed for the filename generation methods provided by the base - # class, CCompiler. NB. whoever instantiates/uses a particular - # UnixCCompiler instance should set 'shared_lib_ext' -- we set a - # reasonable common default here, but it's not necessarily used on all - # Unices! - - src_extensions = [".c",".C",".cc",".cxx",".cpp",".m"] - obj_extension = ".o" - static_lib_extension = ".a" - shared_lib_extension = ".so" - dylib_lib_extension = ".dylib" - xcode_stub_lib_extension = ".tbd" - static_lib_format = shared_lib_format = dylib_lib_format = "lib%s%s" - xcode_stub_lib_format = dylib_lib_format - if sys.platform == "cygwin": - exe_extension = ".exe" - - def preprocess(self, source, output_file=None, macros=None, - include_dirs=None, extra_preargs=None, extra_postargs=None): - fixed_args = self._fix_compile_args(None, macros, include_dirs) - ignore, macros, include_dirs = fixed_args - pp_opts = gen_preprocess_options(macros, include_dirs) - pp_args = self.preprocessor + pp_opts - if output_file: - pp_args.extend(['-o', output_file]) - if extra_preargs: - pp_args[:0] = extra_preargs - if extra_postargs: - pp_args.extend(extra_postargs) - pp_args.append(source) - - # We need to preprocess: either we're being forced to, or we're - # generating output to stdout, or there's a target output file and - # the source file is newer than the target (or the target doesn't - # exist). - if self.force or output_file is None or newer(source, output_file): - if output_file: - self.mkpath(os.path.dirname(output_file)) - try: - self.spawn(pp_args) - except DistutilsExecError as msg: - raise CompileError(msg) - - def _compile(self, obj, src, ext, cc_args, extra_postargs, pp_opts): - compiler_so = self.compiler_so - if sys.platform == 'darwin': - compiler_so = _osx_support.compiler_fixup(compiler_so, - cc_args + extra_postargs) - try: - self.spawn(compiler_so + cc_args + [src, '-o', obj] + - extra_postargs) - except DistutilsExecError as msg: - raise CompileError(msg) - - def create_static_lib(self, objects, output_libname, - output_dir=None, debug=0, target_lang=None): - objects, output_dir = self._fix_object_args(objects, output_dir) - - output_filename = \ - self.library_filename(output_libname, output_dir=output_dir) - - if self._need_link(objects, output_filename): - self.mkpath(os.path.dirname(output_filename)) - self.spawn(self.archiver + - [output_filename] + - objects + self.objects) - - # Not many Unices required ranlib anymore -- SunOS 4.x is, I - # think the only major Unix that does. Maybe we need some - # platform intelligence here to skip ranlib if it's not - # needed -- or maybe Python's configure script took care of - # it for us, hence the check for leading colon. - if self.ranlib: - try: - self.spawn(self.ranlib + [output_filename]) - except DistutilsExecError as msg: - raise LibError(msg) - else: - log.debug("skipping %s (up-to-date)", output_filename) - - def link(self, target_desc, objects, - output_filename, output_dir=None, libraries=None, - library_dirs=None, runtime_library_dirs=None, - export_symbols=None, debug=0, extra_preargs=None, - extra_postargs=None, build_temp=None, target_lang=None): - objects, output_dir = self._fix_object_args(objects, output_dir) - fixed_args = self._fix_lib_args(libraries, library_dirs, - runtime_library_dirs) - libraries, library_dirs, runtime_library_dirs = fixed_args - - # filter out standard library paths, which are not explicitely needed - # for linking - system_libdirs = ['/lib', '/lib64', '/usr/lib', '/usr/lib64'] - multiarch = sysconfig.get_config_var("MULTIARCH") - if multiarch: - system_libdirs.extend(['/lib/%s' % multiarch, '/usr/lib/%s' % multiarch]) - library_dirs = [dir for dir in library_dirs - if not dir in system_libdirs] - runtime_library_dirs = [dir for dir in runtime_library_dirs - if not dir in system_libdirs] - - lib_opts = gen_lib_options(self, library_dirs, runtime_library_dirs, - libraries) - if not isinstance(output_dir, (str, type(None))): - raise TypeError("'output_dir' must be a string or None") - if output_dir is not None: - output_filename = os.path.join(output_dir, output_filename) - - if self._need_link(objects, output_filename): - ld_args = (objects + self.objects + - lib_opts + ['-o', output_filename]) - if debug: - ld_args[:0] = ['-g'] - if extra_preargs: - ld_args[:0] = extra_preargs - if extra_postargs: - ld_args.extend(extra_postargs) - self.mkpath(os.path.dirname(output_filename)) - try: - if target_desc == CCompiler.EXECUTABLE: - linker = self.linker_exe[:] - else: - linker = self.linker_so[:] - if target_lang == "c++" and self.compiler_cxx: - # skip over environment variable settings if /usr/bin/env - # is used to set up the linker's environment. - # This is needed on OSX. Note: this assumes that the - # normal and C++ compiler have the same environment - # settings. - i = 0 - if os.path.basename(linker[0]) == "env": - i = 1 - while '=' in linker[i]: - i += 1 - linker[i] = self.compiler_cxx[i] - - if sys.platform == 'darwin': - linker = _osx_support.compiler_fixup(linker, ld_args) - - self.spawn(linker + ld_args) - except DistutilsExecError as msg: - raise LinkError(msg) - else: - log.debug("skipping %s (up-to-date)", output_filename) - - # -- Miscellaneous methods ----------------------------------------- - # These are all used by the 'gen_lib_options() function, in - # ccompiler.py. - - def library_dir_option(self, dir): - return "-L" + dir - - def _is_gcc(self, compiler_name): - return "gcc" in compiler_name or "g++" in compiler_name - - def runtime_library_dir_option(self, dir): - # XXX Hackish, at the very least. See Python bug #445902: - # http://sourceforge.net/tracker/index.php - # ?func=detail&aid=445902&group_id=5470&atid=105470 - # Linkers on different platforms need different options to - # specify that directories need to be added to the list of - # directories searched for dependencies when a dynamic library - # is sought. GCC on GNU systems (Linux, FreeBSD, ...) has to - # be told to pass the -R option through to the linker, whereas - # other compilers and gcc on other systems just know this. - # Other compilers may need something slightly different. At - # this time, there's no way to determine this information from - # the configuration data stored in the Python installation, so - # we use this hack. - compiler = os.path.basename(sysconfig.get_config_var("CC")) - if sys.platform[:6] == "darwin": - # MacOSX's linker doesn't understand the -R flag at all - return "-L" + dir - elif sys.platform[:7] == "freebsd": - return "-Wl,-rpath=" + dir - elif sys.platform[:5] == "hp-ux": - if self._is_gcc(compiler): - return ["-Wl,+s", "-L" + dir] - return ["+s", "-L" + dir] - elif sys.platform[:7] == "irix646" or sys.platform[:6] == "osf1V5": - return ["-rpath", dir] - else: - if self._is_gcc(compiler): - # gcc on non-GNU systems does not need -Wl, but can - # use it anyway. Since distutils has always passed in - # -Wl whenever gcc was used in the past it is probably - # safest to keep doing so. - if sysconfig.get_config_var("GNULD") == "yes": - # GNU ld needs an extra option to get a RUNPATH - # instead of just an RPATH. - return "-Wl,--enable-new-dtags,-R" + dir - else: - return "-Wl,-R" + dir - else: - # No idea how --enable-new-dtags would be passed on to - # ld if this system was using GNU ld. Don't know if a - # system like this even exists. - return "-R" + dir - - def library_option(self, lib): - return "-l" + lib - - def find_library_file(self, dirs, lib, debug=0): - shared_f = self.library_filename(lib, lib_type='shared') - dylib_f = self.library_filename(lib, lib_type='dylib') - xcode_stub_f = self.library_filename(lib, lib_type='xcode_stub') - static_f = self.library_filename(lib, lib_type='static') - - if sys.platform == 'darwin': - # On OSX users can specify an alternate SDK using - # '-isysroot', calculate the SDK root if it is specified - # (and use it further on) - # - # Note that, as of Xcode 7, Apple SDKs may contain textual stub - # libraries with .tbd extensions rather than the normal .dylib - # shared libraries installed in /. The Apple compiler tool - # chain handles this transparently but it can cause problems - # for programs that are being built with an SDK and searching - # for specific libraries. Callers of find_library_file need to - # keep in mind that the base filename of the returned SDK library - # file might have a different extension from that of the library - # file installed on the running system, for example: - # /Applications/Xcode.app/Contents/Developer/Platforms/ - # MacOSX.platform/Developer/SDKs/MacOSX10.11.sdk/ - # usr/lib/libedit.tbd - # vs - # /usr/lib/libedit.dylib - cflags = sysconfig.get_config_var('CFLAGS') - m = re.search(r'-isysroot\s+(\S+)', cflags) - if m is None: - sysroot = '/' - else: - sysroot = m.group(1) - - - - for dir in dirs: - shared = os.path.join(dir, shared_f) - dylib = os.path.join(dir, dylib_f) - static = os.path.join(dir, static_f) - xcode_stub = os.path.join(dir, xcode_stub_f) - - if sys.platform == 'darwin' and ( - dir.startswith('/System/') or ( - dir.startswith('/usr/') and not dir.startswith('/usr/local/'))): - - shared = os.path.join(sysroot, dir[1:], shared_f) - dylib = os.path.join(sysroot, dir[1:], dylib_f) - static = os.path.join(sysroot, dir[1:], static_f) - xcode_stub = os.path.join(sysroot, dir[1:], xcode_stub_f) - - # We're second-guessing the linker here, with not much hard - # data to go on: GCC seems to prefer the shared library, so I'm - # assuming that *all* Unix C compilers do. And of course I'm - # ignoring even GCC's "-static" option. So sue me. - if os.path.exists(dylib): - return dylib - elif os.path.exists(xcode_stub): - return xcode_stub - elif os.path.exists(shared): - return shared - elif os.path.exists(static): - return static - - # Oops, didn't find it in *any* of 'dirs' - return None diff --git a/Lib/distutils/util.py b/Lib/distutils/util.py deleted file mode 100644 index fdcf6fabae2..00000000000 --- a/Lib/distutils/util.py +++ /dev/null @@ -1,557 +0,0 @@ -"""distutils.util - -Miscellaneous utility functions -- anything that doesn't fit into -one of the other *util.py modules. -""" - -import os -import re -import importlib.util -import string -import sys -from distutils.errors import DistutilsPlatformError -from distutils.dep_util import newer -from distutils.spawn import spawn -from distutils import log -from distutils.errors import DistutilsByteCompileError - -def get_platform (): - """Return a string that identifies the current platform. This is used - mainly to distinguish platform-specific build directories and - platform-specific built distributions. Typically includes the OS name - and version and the architecture (as supplied by 'os.uname()'), - although the exact information included depends on the OS; eg. for IRIX - the architecture isn't particularly important (IRIX only runs on SGI - hardware), but for Linux the kernel version isn't particularly - important. - - Examples of returned values: - linux-i586 - linux-alpha (?) - solaris-2.6-sun4u - irix-5.3 - irix64-6.2 - - Windows will return one of: - win-amd64 (64bit Windows on AMD64 (aka x86_64, Intel64, EM64T, etc) - win-ia64 (64bit Windows on Itanium) - win32 (all others - specifically, sys.platform is returned) - - For other non-POSIX platforms, currently just returns 'sys.platform'. - """ - if os.name == 'nt': - # sniff sys.version for architecture. - prefix = " bit (" - i = sys.version.find(prefix) - if i == -1: - return sys.platform - j = sys.version.find(")", i) - look = sys.version[i+len(prefix):j].lower() - if look == 'amd64': - return 'win-amd64' - if look == 'itanium': - return 'win-ia64' - return sys.platform - - # Set for cross builds explicitly - if "_PYTHON_HOST_PLATFORM" in os.environ: - return os.environ["_PYTHON_HOST_PLATFORM"] - - if os.name != "posix" or not hasattr(os, 'uname'): - # XXX what about the architecture? NT is Intel or Alpha, - # Mac OS is M68k or PPC, etc. - return sys.platform - - # Try to distinguish various flavours of Unix - - (osname, host, release, version, machine) = os.uname() - - # Convert the OS name to lowercase, remove '/' characters - # (to accommodate BSD/OS), and translate spaces (for "Power Macintosh") - osname = osname.lower().replace('/', '') - machine = machine.replace(' ', '_') - machine = machine.replace('/', '-') - - if osname[:5] == "linux": - # At least on Linux/Intel, 'machine' is the processor -- - # i386, etc. - # XXX what about Alpha, SPARC, etc? - return "%s-%s" % (osname, machine) - elif osname[:5] == "sunos": - if release[0] >= "5": # SunOS 5 == Solaris 2 - osname = "solaris" - release = "%d.%s" % (int(release[0]) - 3, release[2:]) - # We can't use "platform.architecture()[0]" because a - # bootstrap problem. We use a dict to get an error - # if some suspicious happens. - bitness = {2147483647:"32bit", 9223372036854775807:"64bit"} - machine += ".%s" % bitness[sys.maxsize] - # fall through to standard osname-release-machine representation - elif osname[:4] == "irix": # could be "irix64"! - return "%s-%s" % (osname, release) - elif osname[:3] == "aix": - return "%s-%s.%s" % (osname, version, release) - elif osname[:6] == "cygwin": - osname = "cygwin" - rel_re = re.compile (r'[\d.]+', re.ASCII) - m = rel_re.match(release) - if m: - release = m.group() - elif osname[:6] == "darwin": - import _osx_support, distutils.sysconfig - osname, release, machine = _osx_support.get_platform_osx( - distutils.sysconfig.get_config_vars(), - osname, release, machine) - - return "%s-%s-%s" % (osname, release, machine) - -# get_platform () - - -def convert_path (pathname): - """Return 'pathname' as a name that will work on the native filesystem, - i.e. split it on '/' and put it back together again using the current - directory separator. Needed because filenames in the setup script are - always supplied in Unix style, and have to be converted to the local - convention before we can actually use them in the filesystem. Raises - ValueError on non-Unix-ish systems if 'pathname' either starts or - ends with a slash. - """ - if os.sep == '/': - return pathname - if not pathname: - return pathname - if pathname[0] == '/': - raise ValueError("path '%s' cannot be absolute" % pathname) - if pathname[-1] == '/': - raise ValueError("path '%s' cannot end with '/'" % pathname) - - paths = pathname.split('/') - while '.' in paths: - paths.remove('.') - if not paths: - return os.curdir - return os.path.join(*paths) - -# convert_path () - - -def change_root (new_root, pathname): - """Return 'pathname' with 'new_root' prepended. If 'pathname' is - relative, this is equivalent to "os.path.join(new_root,pathname)". - Otherwise, it requires making 'pathname' relative and then joining the - two, which is tricky on DOS/Windows and Mac OS. - """ - if os.name == 'posix': - if not os.path.isabs(pathname): - return os.path.join(new_root, pathname) - else: - return os.path.join(new_root, pathname[1:]) - - elif os.name == 'nt': - (drive, path) = os.path.splitdrive(pathname) - if path[0] == '\\': - path = path[1:] - return os.path.join(new_root, path) - - else: - raise DistutilsPlatformError("nothing known about platform '%s'" % os.name) - - -_environ_checked = 0 -def check_environ (): - """Ensure that 'os.environ' has all the environment variables we - guarantee that users can use in config files, command-line options, - etc. Currently this includes: - HOME - user's home directory (Unix only) - PLAT - description of the current platform, including hardware - and OS (see 'get_platform()') - """ - global _environ_checked - if _environ_checked: - return - - if os.name == 'posix' and 'HOME' not in os.environ: - import pwd - os.environ['HOME'] = pwd.getpwuid(os.getuid())[5] - - if 'PLAT' not in os.environ: - os.environ['PLAT'] = get_platform() - - _environ_checked = 1 - - -def subst_vars (s, local_vars): - """Perform shell/Perl-style variable substitution on 'string'. Every - occurrence of '$' followed by a name is considered a variable, and - variable is substituted by the value found in the 'local_vars' - dictionary, or in 'os.environ' if it's not in 'local_vars'. - 'os.environ' is first checked/augmented to guarantee that it contains - certain values: see 'check_environ()'. Raise ValueError for any - variables not found in either 'local_vars' or 'os.environ'. - """ - check_environ() - def _subst (match, local_vars=local_vars): - var_name = match.group(1) - if var_name in local_vars: - return str(local_vars[var_name]) - else: - return os.environ[var_name] - - try: - return re.sub(r'\$([a-zA-Z_][a-zA-Z_0-9]*)', _subst, s) - except KeyError as var: - raise ValueError("invalid variable '$%s'" % var) - -# subst_vars () - - -def grok_environment_error (exc, prefix="error: "): - # Function kept for backward compatibility. - # Used to try clever things with EnvironmentErrors, - # but nowadays str(exception) produces good messages. - return prefix + str(exc) - - -# Needed by 'split_quoted()' -_wordchars_re = _squote_re = _dquote_re = None -def _init_regex(): - global _wordchars_re, _squote_re, _dquote_re - _wordchars_re = re.compile(r'[^\\\'\"%s ]*' % string.whitespace) - _squote_re = re.compile(r"'(?:[^'\\]|\\.)*'") - _dquote_re = re.compile(r'"(?:[^"\\]|\\.)*"') - -def split_quoted (s): - """Split a string up according to Unix shell-like rules for quotes and - backslashes. In short: words are delimited by spaces, as long as those - spaces are not escaped by a backslash, or inside a quoted string. - Single and double quotes are equivalent, and the quote characters can - be backslash-escaped. The backslash is stripped from any two-character - escape sequence, leaving only the escaped character. The quote - characters are stripped from any quoted string. Returns a list of - words. - """ - - # This is a nice algorithm for splitting up a single string, since it - # doesn't require character-by-character examination. It was a little - # bit of a brain-bender to get it working right, though... - if _wordchars_re is None: _init_regex() - - s = s.strip() - words = [] - pos = 0 - - while s: - m = _wordchars_re.match(s, pos) - end = m.end() - if end == len(s): - words.append(s[:end]) - break - - if s[end] in string.whitespace: # unescaped, unquoted whitespace: now - words.append(s[:end]) # we definitely have a word delimiter - s = s[end:].lstrip() - pos = 0 - - elif s[end] == '\\': # preserve whatever is being escaped; - # will become part of the current word - s = s[:end] + s[end+1:] - pos = end+1 - - else: - if s[end] == "'": # slurp singly-quoted string - m = _squote_re.match(s, end) - elif s[end] == '"': # slurp doubly-quoted string - m = _dquote_re.match(s, end) - else: - raise RuntimeError("this can't happen (bad char '%c')" % s[end]) - - if m is None: - raise ValueError("bad string (mismatched %s quotes?)" % s[end]) - - (beg, end) = m.span() - s = s[:beg] + s[beg+1:end-1] + s[end:] - pos = m.end() - 2 - - if pos >= len(s): - words.append(s) - break - - return words - -# split_quoted () - - -def execute (func, args, msg=None, verbose=0, dry_run=0): - """Perform some action that affects the outside world (eg. by - writing to the filesystem). Such actions are special because they - are disabled by the 'dry_run' flag. This method takes care of all - that bureaucracy for you; all you have to do is supply the - function to call and an argument tuple for it (to embody the - "external action" being performed), and an optional message to - print. - """ - if msg is None: - msg = "%s%r" % (func.__name__, args) - if msg[-2:] == ',)': # correct for singleton tuple - msg = msg[0:-2] + ')' - - log.info(msg) - if not dry_run: - func(*args) - - -def strtobool (val): - """Convert a string representation of truth to true (1) or false (0). - - True values are 'y', 'yes', 't', 'true', 'on', and '1'; false values - are 'n', 'no', 'f', 'false', 'off', and '0'. Raises ValueError if - 'val' is anything else. - """ - val = val.lower() - if val in ('y', 'yes', 't', 'true', 'on', '1'): - return 1 - elif val in ('n', 'no', 'f', 'false', 'off', '0'): - return 0 - else: - raise ValueError("invalid truth value %r" % (val,)) - - -def byte_compile (py_files, - optimize=0, force=0, - prefix=None, base_dir=None, - verbose=1, dry_run=0, - direct=None): - """Byte-compile a collection of Python source files to .pyc - files in a __pycache__ subdirectory. 'py_files' is a list - of files to compile; any files that don't end in ".py" are silently - skipped. 'optimize' must be one of the following: - 0 - don't optimize - 1 - normal optimization (like "python -O") - 2 - extra optimization (like "python -OO") - If 'force' is true, all files are recompiled regardless of - timestamps. - - The source filename encoded in each bytecode file defaults to the - filenames listed in 'py_files'; you can modify these with 'prefix' and - 'basedir'. 'prefix' is a string that will be stripped off of each - source filename, and 'base_dir' is a directory name that will be - prepended (after 'prefix' is stripped). You can supply either or both - (or neither) of 'prefix' and 'base_dir', as you wish. - - If 'dry_run' is true, doesn't actually do anything that would - affect the filesystem. - - Byte-compilation is either done directly in this interpreter process - with the standard py_compile module, or indirectly by writing a - temporary script and executing it. Normally, you should let - 'byte_compile()' figure out to use direct compilation or not (see - the source for details). The 'direct' flag is used by the script - generated in indirect mode; unless you know what you're doing, leave - it set to None. - """ - - # Late import to fix a bootstrap issue: _posixsubprocess is built by - # setup.py, but setup.py uses distutils. - import subprocess - - # nothing is done if sys.dont_write_bytecode is True - if sys.dont_write_bytecode: - raise DistutilsByteCompileError('byte-compiling is disabled.') - - # First, if the caller didn't force us into direct or indirect mode, - # figure out which mode we should be in. We take a conservative - # approach: choose direct mode *only* if the current interpreter is - # in debug mode and optimize is 0. If we're not in debug mode (-O - # or -OO), we don't know which level of optimization this - # interpreter is running with, so we can't do direct - # byte-compilation and be certain that it's the right thing. Thus, - # always compile indirectly if the current interpreter is in either - # optimize mode, or if either optimization level was requested by - # the caller. - if direct is None: - direct = (__debug__ and optimize == 0) - - # "Indirect" byte-compilation: write a temporary script and then - # run it with the appropriate flags. - if not direct: - try: - from tempfile import mkstemp - (script_fd, script_name) = mkstemp(".py") - except ImportError: - from tempfile import mktemp - (script_fd, script_name) = None, mktemp(".py") - log.info("writing byte-compilation script '%s'", script_name) - if not dry_run: - if script_fd is not None: - script = os.fdopen(script_fd, "w") - else: - script = open(script_name, "w") - - script.write("""\ -from distutils.util import byte_compile -files = [ -""") - - # XXX would be nice to write absolute filenames, just for - # safety's sake (script should be more robust in the face of - # chdir'ing before running it). But this requires abspath'ing - # 'prefix' as well, and that breaks the hack in build_lib's - # 'byte_compile()' method that carefully tacks on a trailing - # slash (os.sep really) to make sure the prefix here is "just - # right". This whole prefix business is rather delicate -- the - # problem is that it's really a directory, but I'm treating it - # as a dumb string, so trailing slashes and so forth matter. - - #py_files = map(os.path.abspath, py_files) - #if prefix: - # prefix = os.path.abspath(prefix) - - script.write(",\n".join(map(repr, py_files)) + "]\n") - script.write(""" -byte_compile(files, optimize=%r, force=%r, - prefix=%r, base_dir=%r, - verbose=%r, dry_run=0, - direct=1) -""" % (optimize, force, prefix, base_dir, verbose)) - - script.close() - - cmd = [sys.executable] - cmd.extend(subprocess._optim_args_from_interpreter_flags()) - cmd.append(script_name) - spawn(cmd, dry_run=dry_run) - execute(os.remove, (script_name,), "removing %s" % script_name, - dry_run=dry_run) - - # "Direct" byte-compilation: use the py_compile module to compile - # right here, right now. Note that the script generated in indirect - # mode simply calls 'byte_compile()' in direct mode, a weird sort of - # cross-process recursion. Hey, it works! - else: - from py_compile import compile - - for file in py_files: - if file[-3:] != ".py": - # This lets us be lazy and not filter filenames in - # the "install_lib" command. - continue - - # Terminology from the py_compile module: - # cfile - byte-compiled file - # dfile - purported source filename (same as 'file' by default) - if optimize >= 0: - opt = '' if optimize == 0 else optimize - cfile = importlib.util.cache_from_source( - file, optimization=opt) - else: - cfile = importlib.util.cache_from_source(file) - dfile = file - if prefix: - if file[:len(prefix)] != prefix: - raise ValueError("invalid prefix: filename %r doesn't start with %r" - % (file, prefix)) - dfile = dfile[len(prefix):] - if base_dir: - dfile = os.path.join(base_dir, dfile) - - cfile_base = os.path.basename(cfile) - if direct: - if force or newer(file, cfile): - log.info("byte-compiling %s to %s", file, cfile_base) - if not dry_run: - compile(file, cfile, dfile) - else: - log.debug("skipping byte-compilation of %s to %s", - file, cfile_base) - -# byte_compile () - -def rfc822_escape (header): - """Return a version of the string escaped for inclusion in an - RFC-822 header, by ensuring there are 8 spaces space after each newline. - """ - lines = header.split('\n') - sep = '\n' + 8 * ' ' - return sep.join(lines) - -# 2to3 support - -def run_2to3(files, fixer_names=None, options=None, explicit=None): - """Invoke 2to3 on a list of Python files. - The files should all come from the build area, as the - modification is done in-place. To reduce the build time, - only files modified since the last invocation of this - function should be passed in the files argument.""" - - if not files: - return - - # Make this class local, to delay import of 2to3 - from lib2to3.refactor import RefactoringTool, get_fixers_from_package - class DistutilsRefactoringTool(RefactoringTool): - def log_error(self, msg, *args, **kw): - log.error(msg, *args) - - def log_message(self, msg, *args): - log.info(msg, *args) - - def log_debug(self, msg, *args): - log.debug(msg, *args) - - if fixer_names is None: - fixer_names = get_fixers_from_package('lib2to3.fixes') - r = DistutilsRefactoringTool(fixer_names, options=options) - r.refactor(files, write=True) - -def copydir_run_2to3(src, dest, template=None, fixer_names=None, - options=None, explicit=None): - """Recursively copy a directory, only copying new and changed files, - running run_2to3 over all newly copied Python modules afterward. - - If you give a template string, it's parsed like a MANIFEST.in. - """ - from distutils.dir_util import mkpath - from distutils.file_util import copy_file - from distutils.filelist import FileList - filelist = FileList() - curdir = os.getcwd() - os.chdir(src) - try: - filelist.findall() - finally: - os.chdir(curdir) - filelist.files[:] = filelist.allfiles - if template: - for line in template.splitlines(): - line = line.strip() - if not line: continue - filelist.process_template_line(line) - copied = [] - for filename in filelist.files: - outname = os.path.join(dest, filename) - mkpath(os.path.dirname(outname)) - res = copy_file(os.path.join(src, filename), outname, update=1) - if res[1]: copied.append(outname) - run_2to3([fn for fn in copied if fn.lower().endswith('.py')], - fixer_names=fixer_names, options=options, explicit=explicit) - return copied - -class Mixin2to3: - '''Mixin class for commands that run 2to3. - To configure 2to3, setup scripts may either change - the class variables, or inherit from individual commands - to override how 2to3 is invoked.''' - - # provide list of fixers to run; - # defaults to all from lib2to3.fixers - fixer_names = None - - # options dictionary - options = None - - # list of fixers to invoke even though they are marked as explicit - explicit = None - - def run_2to3(self, files): - return run_2to3(files, self.fixer_names, self.options, self.explicit) diff --git a/Lib/distutils/version.py b/Lib/distutils/version.py deleted file mode 100644 index af14cc13481..00000000000 --- a/Lib/distutils/version.py +++ /dev/null @@ -1,343 +0,0 @@ -# -# distutils/version.py -# -# Implements multiple version numbering conventions for the -# Python Module Distribution Utilities. -# -# $Id$ -# - -"""Provides classes to represent module version numbers (one class for -each style of version numbering). There are currently two such classes -implemented: StrictVersion and LooseVersion. - -Every version number class implements the following interface: - * the 'parse' method takes a string and parses it to some internal - representation; if the string is an invalid version number, - 'parse' raises a ValueError exception - * the class constructor takes an optional string argument which, - if supplied, is passed to 'parse' - * __str__ reconstructs the string that was passed to 'parse' (or - an equivalent string -- ie. one that will generate an equivalent - version number instance) - * __repr__ generates Python code to recreate the version number instance - * _cmp compares the current instance with either another instance - of the same class or a string (which will be parsed to an instance - of the same class, thus must follow the same rules) -""" - -import re - -class Version: - """Abstract base class for version numbering classes. Just provides - constructor (__init__) and reproducer (__repr__), because those - seem to be the same for all version numbering classes; and route - rich comparisons to _cmp. - """ - - def __init__ (self, vstring=None): - if vstring: - self.parse(vstring) - - def __repr__ (self): - return "%s ('%s')" % (self.__class__.__name__, str(self)) - - def __eq__(self, other): - c = self._cmp(other) - if c is NotImplemented: - return c - return c == 0 - - def __lt__(self, other): - c = self._cmp(other) - if c is NotImplemented: - return c - return c < 0 - - def __le__(self, other): - c = self._cmp(other) - if c is NotImplemented: - return c - return c <= 0 - - def __gt__(self, other): - c = self._cmp(other) - if c is NotImplemented: - return c - return c > 0 - - def __ge__(self, other): - c = self._cmp(other) - if c is NotImplemented: - return c - return c >= 0 - - -# Interface for version-number classes -- must be implemented -# by the following classes (the concrete ones -- Version should -# be treated as an abstract class). -# __init__ (string) - create and take same action as 'parse' -# (string parameter is optional) -# parse (string) - convert a string representation to whatever -# internal representation is appropriate for -# this style of version numbering -# __str__ (self) - convert back to a string; should be very similar -# (if not identical to) the string supplied to parse -# __repr__ (self) - generate Python code to recreate -# the instance -# _cmp (self, other) - compare two version numbers ('other' may -# be an unparsed version string, or another -# instance of your version class) - - -class StrictVersion (Version): - - """Version numbering for anal retentives and software idealists. - Implements the standard interface for version number classes as - described above. A version number consists of two or three - dot-separated numeric components, with an optional "pre-release" tag - on the end. The pre-release tag consists of the letter 'a' or 'b' - followed by a number. If the numeric components of two version - numbers are equal, then one with a pre-release tag will always - be deemed earlier (lesser) than one without. - - The following are valid version numbers (shown in the order that - would be obtained by sorting according to the supplied cmp function): - - 0.4 0.4.0 (these two are equivalent) - 0.4.1 - 0.5a1 - 0.5b3 - 0.5 - 0.9.6 - 1.0 - 1.0.4a3 - 1.0.4b1 - 1.0.4 - - The following are examples of invalid version numbers: - - 1 - 2.7.2.2 - 1.3.a4 - 1.3pl1 - 1.3c4 - - The rationale for this version numbering system will be explained - in the distutils documentation. - """ - - version_re = re.compile(r'^(\d+) \. (\d+) (\. (\d+))? ([ab](\d+))?$', - re.VERBOSE | re.ASCII) - - - def parse (self, vstring): - match = self.version_re.match(vstring) - if not match: - raise ValueError("invalid version number '%s'" % vstring) - - (major, minor, patch, prerelease, prerelease_num) = \ - match.group(1, 2, 4, 5, 6) - - if patch: - self.version = tuple(map(int, [major, minor, patch])) - else: - self.version = tuple(map(int, [major, minor])) + (0,) - - if prerelease: - self.prerelease = (prerelease[0], int(prerelease_num)) - else: - self.prerelease = None - - - def __str__ (self): - - if self.version[2] == 0: - vstring = '.'.join(map(str, self.version[0:2])) - else: - vstring = '.'.join(map(str, self.version)) - - if self.prerelease: - vstring = vstring + self.prerelease[0] + str(self.prerelease[1]) - - return vstring - - - def _cmp (self, other): - if isinstance(other, str): - other = StrictVersion(other) - - if self.version != other.version: - # numeric versions don't match - # prerelease stuff doesn't matter - if self.version < other.version: - return -1 - else: - return 1 - - # have to compare prerelease - # case 1: neither has prerelease; they're equal - # case 2: self has prerelease, other doesn't; other is greater - # case 3: self doesn't have prerelease, other does: self is greater - # case 4: both have prerelease: must compare them! - - if (not self.prerelease and not other.prerelease): - return 0 - elif (self.prerelease and not other.prerelease): - return -1 - elif (not self.prerelease and other.prerelease): - return 1 - elif (self.prerelease and other.prerelease): - if self.prerelease == other.prerelease: - return 0 - elif self.prerelease < other.prerelease: - return -1 - else: - return 1 - else: - assert False, "never get here" - -# end class StrictVersion - - -# The rules according to Greg Stein: -# 1) a version number has 1 or more numbers separated by a period or by -# sequences of letters. If only periods, then these are compared -# left-to-right to determine an ordering. -# 2) sequences of letters are part of the tuple for comparison and are -# compared lexicographically -# 3) recognize the numeric components may have leading zeroes -# -# The LooseVersion class below implements these rules: a version number -# string is split up into a tuple of integer and string components, and -# comparison is a simple tuple comparison. This means that version -# numbers behave in a predictable and obvious way, but a way that might -# not necessarily be how people *want* version numbers to behave. There -# wouldn't be a problem if people could stick to purely numeric version -# numbers: just split on period and compare the numbers as tuples. -# However, people insist on putting letters into their version numbers; -# the most common purpose seems to be: -# - indicating a "pre-release" version -# ('alpha', 'beta', 'a', 'b', 'pre', 'p') -# - indicating a post-release patch ('p', 'pl', 'patch') -# but of course this can't cover all version number schemes, and there's -# no way to know what a programmer means without asking him. -# -# The problem is what to do with letters (and other non-numeric -# characters) in a version number. The current implementation does the -# obvious and predictable thing: keep them as strings and compare -# lexically within a tuple comparison. This has the desired effect if -# an appended letter sequence implies something "post-release": -# eg. "0.99" < "0.99pl14" < "1.0", and "5.001" < "5.001m" < "5.002". -# -# However, if letters in a version number imply a pre-release version, -# the "obvious" thing isn't correct. Eg. you would expect that -# "1.5.1" < "1.5.2a2" < "1.5.2", but under the tuple/lexical comparison -# implemented here, this just isn't so. -# -# Two possible solutions come to mind. The first is to tie the -# comparison algorithm to a particular set of semantic rules, as has -# been done in the StrictVersion class above. This works great as long -# as everyone can go along with bondage and discipline. Hopefully a -# (large) subset of Python module programmers will agree that the -# particular flavour of bondage and discipline provided by StrictVersion -# provides enough benefit to be worth using, and will submit their -# version numbering scheme to its domination. The free-thinking -# anarchists in the lot will never give in, though, and something needs -# to be done to accommodate them. -# -# Perhaps a "moderately strict" version class could be implemented that -# lets almost anything slide (syntactically), and makes some heuristic -# assumptions about non-digits in version number strings. This could -# sink into special-case-hell, though; if I was as talented and -# idiosyncratic as Larry Wall, I'd go ahead and implement a class that -# somehow knows that "1.2.1" < "1.2.2a2" < "1.2.2" < "1.2.2pl3", and is -# just as happy dealing with things like "2g6" and "1.13++". I don't -# think I'm smart enough to do it right though. -# -# In any case, I've coded the test suite for this module (see -# ../test/test_version.py) specifically to fail on things like comparing -# "1.2a2" and "1.2". That's not because the *code* is doing anything -# wrong, it's because the simple, obvious design doesn't match my -# complicated, hairy expectations for real-world version numbers. It -# would be a snap to fix the test suite to say, "Yep, LooseVersion does -# the Right Thing" (ie. the code matches the conception). But I'd rather -# have a conception that matches common notions about version numbers. - -class LooseVersion (Version): - - """Version numbering for anarchists and software realists. - Implements the standard interface for version number classes as - described above. A version number consists of a series of numbers, - separated by either periods or strings of letters. When comparing - version numbers, the numeric components will be compared - numerically, and the alphabetic components lexically. The following - are all valid version numbers, in no particular order: - - 1.5.1 - 1.5.2b2 - 161 - 3.10a - 8.02 - 3.4j - 1996.07.12 - 3.2.pl0 - 3.1.1.6 - 2g6 - 11g - 0.960923 - 2.2beta29 - 1.13++ - 5.5.kw - 2.0b1pl0 - - In fact, there is no such thing as an invalid version number under - this scheme; the rules for comparison are simple and predictable, - but may not always give the results you want (for some definition - of "want"). - """ - - component_re = re.compile(r'(\d+ | [a-z]+ | \.)', re.VERBOSE) - - def __init__ (self, vstring=None): - if vstring: - self.parse(vstring) - - - def parse (self, vstring): - # I've given up on thinking I can reconstruct the version string - # from the parsed tuple -- so I just store the string here for - # use by __str__ - self.vstring = vstring - components = [x for x in self.component_re.split(vstring) - if x and x != '.'] - for i, obj in enumerate(components): - try: - components[i] = int(obj) - except ValueError: - pass - - self.version = components - - - def __str__ (self): - return self.vstring - - - def __repr__ (self): - return "LooseVersion ('%s')" % str(self) - - - def _cmp (self, other): - if isinstance(other, str): - other = LooseVersion(other) - - if self.version == other.version: - return 0 - if self.version < other.version: - return -1 - if self.version > other.version: - return 1 - - -# end class LooseVersion diff --git a/Lib/distutils/versionpredicate.py b/Lib/distutils/versionpredicate.py deleted file mode 100644 index 062c98f2489..00000000000 --- a/Lib/distutils/versionpredicate.py +++ /dev/null @@ -1,166 +0,0 @@ -"""Module for parsing and testing package version predicate strings. -""" -import re -import distutils.version -import operator - - -re_validPackage = re.compile(r"(?i)^\s*([a-z_]\w*(?:\.[a-z_]\w*)*)(.*)", - re.ASCII) -# (package) (rest) - -re_paren = re.compile(r"^\s*\((.*)\)\s*$") # (list) inside of parentheses -re_splitComparison = re.compile(r"^\s*(<=|>=|<|>|!=|==)\s*([^\s,]+)\s*$") -# (comp) (version) - - -def splitUp(pred): - """Parse a single version comparison. - - Return (comparison string, StrictVersion) - """ - res = re_splitComparison.match(pred) - if not res: - raise ValueError("bad package restriction syntax: %r" % pred) - comp, verStr = res.groups() - return (comp, distutils.version.StrictVersion(verStr)) - -compmap = {"<": operator.lt, "<=": operator.le, "==": operator.eq, - ">": operator.gt, ">=": operator.ge, "!=": operator.ne} - -class VersionPredicate: - """Parse and test package version predicates. - - >>> v = VersionPredicate('pyepat.abc (>1.0, <3333.3a1, !=1555.1b3)') - - The `name` attribute provides the full dotted name that is given:: - - >>> v.name - 'pyepat.abc' - - The str() of a `VersionPredicate` provides a normalized - human-readable version of the expression:: - - >>> print(v) - pyepat.abc (> 1.0, < 3333.3a1, != 1555.1b3) - - The `satisfied_by()` method can be used to determine with a given - version number is included in the set described by the version - restrictions:: - - >>> v.satisfied_by('1.1') - True - >>> v.satisfied_by('1.4') - True - >>> v.satisfied_by('1.0') - False - >>> v.satisfied_by('4444.4') - False - >>> v.satisfied_by('1555.1b3') - False - - `VersionPredicate` is flexible in accepting extra whitespace:: - - >>> v = VersionPredicate(' pat( == 0.1 ) ') - >>> v.name - 'pat' - >>> v.satisfied_by('0.1') - True - >>> v.satisfied_by('0.2') - False - - If any version numbers passed in do not conform to the - restrictions of `StrictVersion`, a `ValueError` is raised:: - - >>> v = VersionPredicate('p1.p2.p3.p4(>=1.0, <=1.3a1, !=1.2zb3)') - Traceback (most recent call last): - ... - ValueError: invalid version number '1.2zb3' - - It the module or package name given does not conform to what's - allowed as a legal module or package name, `ValueError` is - raised:: - - >>> v = VersionPredicate('foo-bar') - Traceback (most recent call last): - ... - ValueError: expected parenthesized list: '-bar' - - >>> v = VersionPredicate('foo bar (12.21)') - Traceback (most recent call last): - ... - ValueError: expected parenthesized list: 'bar (12.21)' - - """ - - def __init__(self, versionPredicateStr): - """Parse a version predicate string. - """ - # Fields: - # name: package name - # pred: list of (comparison string, StrictVersion) - - versionPredicateStr = versionPredicateStr.strip() - if not versionPredicateStr: - raise ValueError("empty package restriction") - match = re_validPackage.match(versionPredicateStr) - if not match: - raise ValueError("bad package name in %r" % versionPredicateStr) - self.name, paren = match.groups() - paren = paren.strip() - if paren: - match = re_paren.match(paren) - if not match: - raise ValueError("expected parenthesized list: %r" % paren) - str = match.groups()[0] - self.pred = [splitUp(aPred) for aPred in str.split(",")] - if not self.pred: - raise ValueError("empty parenthesized list in %r" - % versionPredicateStr) - else: - self.pred = [] - - def __str__(self): - if self.pred: - seq = [cond + " " + str(ver) for cond, ver in self.pred] - return self.name + " (" + ", ".join(seq) + ")" - else: - return self.name - - def satisfied_by(self, version): - """True if version is compatible with all the predicates in self. - The parameter version must be acceptable to the StrictVersion - constructor. It may be either a string or StrictVersion. - """ - for cond, ver in self.pred: - if not compmap[cond](version, ver): - return False - return True - - -_provision_rx = None - -def split_provision(value): - """Return the name and optional version number of a provision. - - The version number, if given, will be returned as a `StrictVersion` - instance, otherwise it will be `None`. - - >>> split_provision('mypkg') - ('mypkg', None) - >>> split_provision(' mypkg( 1.2 ) ') - ('mypkg', StrictVersion ('1.2')) - """ - global _provision_rx - if _provision_rx is None: - _provision_rx = re.compile( - r"([a-zA-Z_]\w*(?:\.[a-zA-Z_]\w*)*)(?:\s*\(\s*([^)\s]+)\s*\))?$", - re.ASCII) - value = value.strip() - m = _provision_rx.match(value) - if not m: - raise ValueError("illegal provides specification: %r" % value) - ver = m.group(2) or None - if ver: - ver = distutils.version.StrictVersion(ver) - return m.group(1), ver diff --git a/Lib/doctest.py b/Lib/doctest.py index 387f71b184a..ecac54ad5a5 100644 --- a/Lib/doctest.py +++ b/Lib/doctest.py @@ -94,6 +94,7 @@ def _test(): import __future__ import difflib +import functools import inspect import linecache import os @@ -104,8 +105,26 @@ def _test(): import unittest from io import StringIO, IncrementalNewlineDecoder from collections import namedtuple +import _colorize # Used in doctests +from _colorize import ANSIColors, can_colorize + + +class TestResults(namedtuple('TestResults', 'failed attempted')): + def __new__(cls, failed, attempted, *, skipped=0): + results = super().__new__(cls, failed, attempted) + results.skipped = skipped + return results + + def __repr__(self): + if self.skipped: + return (f'TestResults(failed={self.failed}, ' + f'attempted={self.attempted}, ' + f'skipped={self.skipped})') + else: + # Leave the repr() unchanged for backward compatibility + # if skipped is zero + return super().__repr__() -TestResults = namedtuple('TestResults', 'failed attempted') # There are 4 basic classes: # - Example: a pair, plus an intra-docstring line number. @@ -207,7 +226,13 @@ def _normalize_module(module, depth=2): elif isinstance(module, str): return __import__(module, globals(), locals(), ["*"]) elif module is None: - return sys.modules[sys._getframe(depth).f_globals['__name__']] + try: + try: + return sys.modules[sys._getframemodulename(depth)] + except AttributeError: + return sys.modules[sys._getframe(depth).f_globals['__name__']] + except KeyError: + pass else: raise TypeError("Expected a module, string, or None") @@ -229,7 +254,6 @@ def _load_testfile(filename, package, module_relative, encoding): file_contents = file_contents.decode(encoding) # get_data() opens files as 'rb', so one must do the equivalent # conversion as universal newlines would do. - return _newline_convert(file_contents), filename with open(filename, encoding=encoding) as f: return f.read(), filename @@ -570,9 +594,11 @@ def __hash__(self): def __lt__(self, other): if not isinstance(other, DocTest): return NotImplemented - return ((self.name, self.filename, self.lineno, id(self)) + self_lno = self.lineno if self.lineno is not None else -1 + other_lno = other.lineno if other.lineno is not None else -1 + return ((self.name, self.filename, self_lno, id(self)) < - (other.name, other.filename, other.lineno, id(other))) + (other.name, other.filename, other_lno, id(other))) ###################################################################### ## 3. DocTestParser @@ -957,7 +983,8 @@ def _from_module(self, module, object): return module is inspect.getmodule(object) elif inspect.isfunction(object): return module.__dict__ is object.__globals__ - elif inspect.ismethoddescriptor(object): + elif (inspect.ismethoddescriptor(object) or + inspect.ismethodwrapper(object)): if hasattr(object, '__objclass__'): obj_mod = object.__objclass__.__module__ elif hasattr(object, '__module__'): @@ -1104,7 +1131,7 @@ def _find_lineno(self, obj, source_lines): if source_lines is None: return None pat = re.compile(r'^\s*class\s*%s\b' % - getattr(obj, '__name__', '-')) + re.escape(getattr(obj, '__name__', '-'))) for i, line in enumerate(source_lines): if pat.match(line): lineno = i @@ -1112,13 +1139,24 @@ def _find_lineno(self, obj, source_lines): # Find the line number for functions & methods. if inspect.ismethod(obj): obj = obj.__func__ - if inspect.isfunction(obj) and getattr(obj, '__doc__', None): + if isinstance(obj, property): + obj = obj.fget + if isinstance(obj, functools.cached_property): + obj = obj.func + if inspect.isroutine(obj) and getattr(obj, '__doc__', None): # We don't use `docstring` var here, because `obj` can be changed. - obj = obj.__code__ + obj = inspect.unwrap(obj) + try: + obj = obj.__code__ + except AttributeError: + # Functions implemented in C don't necessarily + # have a __code__ attribute. + # If there's no code, there's no lineno + return None if inspect.istraceback(obj): obj = obj.tb_frame if inspect.isframe(obj): obj = obj.f_code if inspect.iscode(obj): - lineno = getattr(obj, 'co_firstlineno', None)-1 + lineno = obj.co_firstlineno - 1 # Find the line number where the docstring starts. Assume # that it's the first line that begins with a quote mark. @@ -1144,8 +1182,10 @@ class DocTestRunner: """ A class used to run DocTest test cases, and accumulate statistics. The `run` method is used to process a single DocTest case. It - returns a tuple `(f, t)`, where `t` is the number of test cases - tried, and `f` is the number of test cases that failed. + returns a TestResults instance. + + >>> save_colorize = _colorize.COLORIZE + >>> _colorize.COLORIZE = False >>> tests = DocTestFinder().find(_TestClass) >>> runner = DocTestRunner(verbose=False) @@ -1158,27 +1198,29 @@ class DocTestRunner: _TestClass.square -> TestResults(failed=0, attempted=1) The `summarize` method prints a summary of all the test cases that - have been run by the runner, and returns an aggregated `(f, t)` - tuple: + have been run by the runner, and returns an aggregated TestResults + instance: >>> runner.summarize(verbose=1) 4 items passed all tests: 2 tests in _TestClass 2 tests in _TestClass.__init__ 2 tests in _TestClass.get - 1 tests in _TestClass.square + 1 test in _TestClass.square 7 tests in 4 items. - 7 passed and 0 failed. + 7 passed. Test passed. TestResults(failed=0, attempted=7) - The aggregated number of tried examples and failed examples is - also available via the `tries` and `failures` attributes: + The aggregated number of tried examples and failed examples is also + available via the `tries`, `failures` and `skips` attributes: >>> runner.tries 7 >>> runner.failures 0 + >>> runner.skips + 0 The comparison between expected outputs and actual outputs is done by an `OutputChecker`. This comparison may be customized with a @@ -1195,6 +1237,8 @@ class DocTestRunner: can be also customized by subclassing DocTestRunner, and overriding the methods `report_start`, `report_success`, `report_unexpected_exception`, and `report_failure`. + + >>> _colorize.COLORIZE = save_colorize """ # This divider string is used to separate failure messages, and to # separate sections of the summary. @@ -1227,7 +1271,8 @@ def __init__(self, checker=None, verbose=None, optionflags=0): # Keep track of the examples we've run. self.tries = 0 self.failures = 0 - self._name2ft = {} + self.skips = 0 + self._stats = {} # Create a fake output target for capturing doctest output. self._fakeout = _SpoofOut() @@ -1272,7 +1317,10 @@ def report_unexpected_exception(self, out, test, example, exc_info): 'Exception raised:\n' + _indent(_exception_traceback(exc_info))) def _failure_header(self, test, example): - out = [self.DIVIDER] + red, reset = ( + (ANSIColors.RED, ANSIColors.RESET) if can_colorize() else ("", "") + ) + out = [f"{red}{self.DIVIDER}{reset}"] if test.filename: if test.lineno is not None and example.lineno is not None: lineno = test.lineno + example.lineno + 1 @@ -1296,13 +1344,11 @@ def __run(self, test, compileflags, out): Run the examples in `test`. Write the outcome of each example with one of the `DocTestRunner.report_*` methods, using the writer function `out`. `compileflags` is the set of compiler - flags that should be used to execute examples. Return a tuple - `(f, t)`, where `t` is the number of examples tried, and `f` - is the number of examples that failed. The examples are run - in the namespace `test.globs`. + flags that should be used to execute examples. Return a TestResults + instance. The examples are run in the namespace `test.globs`. """ - # Keep track of the number of failures and tries. - failures = tries = 0 + # Keep track of the number of failed, attempted, skipped examples. + failures = attempted = skips = 0 # Save the option flags (since option directives can be used # to modify them). @@ -1314,6 +1360,7 @@ def __run(self, test, compileflags, out): # Process each example. for examplenum, example in enumerate(test.examples): + attempted += 1 # If REPORT_ONLY_FIRST_FAILURE is set, then suppress # reporting after the first failure. @@ -1331,10 +1378,10 @@ def __run(self, test, compileflags, out): # If 'SKIP' is set, then skip this example. if self.optionflags & SKIP: + skips += 1 continue # Record that we started this example. - tries += 1 if not quiet: self.report_start(out, test, example) @@ -1370,7 +1417,24 @@ def __run(self, test, compileflags, out): # The example raised an exception: check if it was expected. else: - exc_msg = traceback.format_exception_only(*exception[:2])[-1] + formatted_ex = traceback.format_exception_only(*exception[:2]) + if issubclass(exception[0], SyntaxError): + # SyntaxError / IndentationError is special: + # we don't care about the carets / suggestions / etc + # We only care about the error message and notes. + # They start with `SyntaxError:` (or any other class name) + exception_line_prefixes = ( + f"{exception[0].__qualname__}:", + f"{exception[0].__module__}.{exception[0].__qualname__}:", + ) + exc_msg_index = next( + index + for index, line in enumerate(formatted_ex) + if line.startswith(exception_line_prefixes) + ) + formatted_ex = formatted_ex[exc_msg_index:] + + exc_msg = "".join(formatted_ex) if not quiet: got += _exception_traceback(exception) @@ -1412,19 +1476,22 @@ def __run(self, test, compileflags, out): # Restore the option flags (in case they were modified) self.optionflags = original_optionflags - # Record and return the number of failures and tries. - self.__record_outcome(test, failures, tries) - return TestResults(failures, tries) + # Record and return the number of failures and attempted. + self.__record_outcome(test, failures, attempted, skips) + return TestResults(failures, attempted, skipped=skips) - def __record_outcome(self, test, f, t): + def __record_outcome(self, test, failures, tries, skips): """ - Record the fact that the given DocTest (`test`) generated `f` - failures out of `t` tried examples. + Record the fact that the given DocTest (`test`) generated `failures` + failures out of `tries` tried examples. """ - f2, t2 = self._name2ft.get(test.name, (0,0)) - self._name2ft[test.name] = (f+f2, t+t2) - self.failures += f - self.tries += t + failures2, tries2, skips2 = self._stats.get(test.name, (0, 0, 0)) + self._stats[test.name] = (failures + failures2, + tries + tries2, + skips + skips2) + self.failures += failures + self.tries += tries + self.skips += skips __LINECACHE_FILENAME_RE = re.compile(r'.+)' @@ -1493,7 +1560,11 @@ def out(s): # Make sure sys.displayhook just prints the value to stdout save_displayhook = sys.displayhook sys.displayhook = sys.__displayhook__ - + saved_can_colorize = _colorize.can_colorize + _colorize.can_colorize = lambda *args, **kwargs: False + color_variables = {"PYTHON_COLORS": None, "FORCE_COLOR": None} + for key in color_variables: + color_variables[key] = os.environ.pop(key, None) try: return self.__run(test, compileflags, out) finally: @@ -1502,6 +1573,10 @@ def out(s): sys.settrace(save_trace) linecache.getlines = self.save_linecache_getlines sys.displayhook = save_displayhook + _colorize.can_colorize = saved_can_colorize + for key, value in color_variables.items(): + if value is not None: + os.environ[key] = value if clear_globs: test.globs.clear() import builtins @@ -1513,9 +1588,7 @@ def out(s): def summarize(self, verbose=None): """ Print a summary of all the test cases that have been run by - this DocTestRunner, and return a tuple `(f, t)`, where `f` is - the total number of failed examples, and `t` is the total - number of tried examples. + this DocTestRunner, and return a TestResults instance. The optional `verbose` argument controls how detailed the summary is. If the verbosity is not specified, then the @@ -1523,66 +1596,98 @@ def summarize(self, verbose=None): """ if verbose is None: verbose = self._verbose - notests = [] - passed = [] - failed = [] - totalt = totalf = 0 - for x in self._name2ft.items(): - name, (f, t) = x - assert f <= t - totalt += t - totalf += f - if t == 0: + + notests, passed, failed = [], [], [] + total_tries = total_failures = total_skips = 0 + + for name, (failures, tries, skips) in self._stats.items(): + assert failures <= tries + total_tries += tries + total_failures += failures + total_skips += skips + + if tries == 0: notests.append(name) - elif f == 0: - passed.append( (name, t) ) + elif failures == 0: + passed.append((name, tries)) else: - failed.append(x) + failed.append((name, (failures, tries, skips))) + + ansi = _colorize.get_colors() + bold_green = ansi.BOLD_GREEN + bold_red = ansi.BOLD_RED + green = ansi.GREEN + red = ansi.RED + reset = ansi.RESET + yellow = ansi.YELLOW + if verbose: if notests: - print(len(notests), "items had no tests:") + print(f"{_n_items(notests)} had no tests:") notests.sort() - for thing in notests: - print(" ", thing) + for name in notests: + print(f" {name}") + if passed: - print(len(passed), "items passed all tests:") - passed.sort() - for thing, count in passed: - print(" %3d tests in %s" % (count, thing)) + print(f"{green}{_n_items(passed)} passed all tests:{reset}") + for name, count in sorted(passed): + s = "" if count == 1 else "s" + print(f" {green}{count:3d} test{s} in {name}{reset}") + if failed: - print(self.DIVIDER) - print(len(failed), "items had failures:") - failed.sort() - for thing, (f, t) in failed: - print(" %3d of %3d in %s" % (f, t, thing)) + print(f"{red}{self.DIVIDER}{reset}") + print(f"{_n_items(failed)} had failures:") + for name, (failures, tries, skips) in sorted(failed): + print(f" {failures:3d} of {tries:3d} in {name}") + if verbose: - print(totalt, "tests in", len(self._name2ft), "items.") - print(totalt - totalf, "passed and", totalf, "failed.") - if totalf: - print("***Test Failed***", totalf, "failures.") + s = "" if total_tries == 1 else "s" + print(f"{total_tries} test{s} in {_n_items(self._stats)}.") + + and_f = ( + f" and {red}{total_failures} failed{reset}" + if total_failures else "" + ) + print(f"{green}{total_tries - total_failures} passed{reset}{and_f}.") + + if total_failures: + s = "" if total_failures == 1 else "s" + msg = f"{bold_red}***Test Failed*** {total_failures} failure{s}{reset}" + if total_skips: + s = "" if total_skips == 1 else "s" + msg = f"{msg} and {yellow}{total_skips} skipped test{s}{reset}" + print(f"{msg}.") elif verbose: - print("Test passed.") - return TestResults(totalf, totalt) + print(f"{bold_green}Test passed.{reset}") + + return TestResults(total_failures, total_tries, skipped=total_skips) #///////////////////////////////////////////////////////////////// # Backward compatibility cruft to maintain doctest.master. #///////////////////////////////////////////////////////////////// def merge(self, other): - d = self._name2ft - for name, (f, t) in other._name2ft.items(): + d = self._stats + for name, (failures, tries, skips) in other._stats.items(): if name in d: - # Don't print here by default, since doing - # so breaks some of the buildbots - #print("*** DocTestRunner.merge: '" + name + "' in both" \ - # " testers; summing outcomes.") - f2, t2 = d[name] - f = f + f2 - t = t + t2 - d[name] = f, t + failures2, tries2, skips2 = d[name] + failures = failures + failures2 + tries = tries + tries2 + skips = skips + skips2 + d[name] = (failures, tries, skips) + + +def _n_items(items: list | dict) -> str: + """ + Helper to pluralise the number of items in a list. + """ + n = len(items) + s = "" if n == 1 else "s" + return f"{n} item{s}" + class OutputChecker: """ - A class used to check the whether the actual output from a doctest + A class used to check whether the actual output from a doctest example matches the expected output. `OutputChecker` defines two methods: `check_output`, which compares a given pair of outputs, and returns true if they match; and `output_difference`, which @@ -1887,8 +1992,8 @@ def testmod(m=None, name=None, globs=None, verbose=None, from module m (or the current module if m is not supplied), starting with m.__doc__. - Also test examples reachable from dict m.__test__ if it exists and is - not None. m.__test__ maps names to functions, classes and strings; + Also test examples reachable from dict m.__test__ if it exists. + m.__test__ maps names to functions, classes and strings; function and class docstrings are tested even if the name is private; strings are tested directly, as if they were docstrings. @@ -1978,7 +2083,8 @@ class doctest.Tester, then merges the results into (or creates) else: master.merge(runner) - return TestResults(runner.failures, runner.tries) + return TestResults(runner.failures, runner.tries, skipped=runner.skips) + def testfile(filename, module_relative=True, name=None, package=None, globs=None, verbose=None, report=True, optionflags=0, @@ -2101,7 +2207,8 @@ class doctest.Tester, then merges the results into (or creates) else: master.merge(runner) - return TestResults(runner.failures, runner.tries) + return TestResults(runner.failures, runner.tries, skipped=runner.skips) + def run_docstring_examples(f, globs, verbose=False, name="NoName", compileflags=None, optionflags=0): @@ -2176,13 +2283,13 @@ def __init__(self, test, optionflags=0, setUp=None, tearDown=None, unittest.TestCase.__init__(self) self._dt_optionflags = optionflags self._dt_checker = checker - self._dt_globs = test.globs.copy() self._dt_test = test self._dt_setUp = setUp self._dt_tearDown = tearDown def setUp(self): test = self._dt_test + self._dt_globs = test.globs.copy() if self._dt_setUp is not None: self._dt_setUp(test) @@ -2213,12 +2320,13 @@ def runTest(self): try: runner.DIVIDER = "-"*70 - failures, tries = runner.run( - test, out=new.write, clear_globs=False) + results = runner.run(test, out=new.write, clear_globs=False) + if results.skipped == results.attempted: + raise unittest.SkipTest("all examples were skipped") finally: sys.stdout = old - if failures: + if results.failed: raise self.failureException(self.format_failure(new.getvalue())) def format_failure(self, err): diff --git a/Lib/test/datetimetester.py b/Lib/test/datetimetester.py index 2636c1fbb98..1bb74c6d969 100644 --- a/Lib/test/datetimetester.py +++ b/Lib/test/datetimetester.py @@ -1,3 +1,6 @@ +# TODO: RustPython +# Skipping some tests because time module has no attribute tzset +# implementation detail specific to cpython """Test date/time type. See https://www.zope.dev/Members/fdrake/DateTimeWiki/TestCases @@ -13,6 +16,7 @@ import re import struct import sys +import textwrap import unittest import warnings @@ -22,6 +26,7 @@ from test import support from test.support import is_resource_enabled, ALWAYS_EQ, LARGEST, SMALLEST +from test.support import script_helper, warnings_helper import datetime as datetime_module from datetime import MINYEAR, MAXYEAR @@ -37,6 +42,10 @@ import _testcapi except ImportError: _testcapi = None +try: + import _interpreters +except ModuleNotFoundError: + _interpreters = None # Needed by test_datetime import _strptime @@ -301,6 +310,10 @@ def test_inheritance(self): self.assertIsInstance(timezone.utc, tzinfo) self.assertIsInstance(self.EST, tzinfo) + def test_cannot_subclass(self): + with self.assertRaises(TypeError): + class MyTimezone(timezone): pass + def test_utcoffset(self): dummy = self.DT for h in [0, 1.5, 12]: @@ -752,6 +765,9 @@ def test_str(self): microseconds=999999)), "999999999 days, 23:59:59.999999") + # test the Doc/library/datetime.rst recipe + eq(f'-({-td(hours=-1)!s})', "-(1:00:00)") + def test_repr(self): name = 'datetime.' + self.theclass.__name__ self.assertEqual(repr(self.theclass(1)), @@ -1331,6 +1347,11 @@ def test_insane_fromtimestamp(self): self.assertRaises(OverflowError, self.theclass.fromtimestamp, insane) + def test_fromtimestamp_with_none_arg(self): + # See gh-120268 for more details + with self.assertRaises(TypeError): + self.theclass.fromtimestamp(None) + def test_today(self): import time @@ -1498,8 +1519,7 @@ def test_strftime(self): # bpo-41260: The parameter was named "fmt" in the pure python impl. t.strftime(format="%f") - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_strftime_trailing_percent(self): # bpo-35066: Make sure trailing '%' doesn't cause datetime's strftime to # complain. Different libcs have different handling of trailing @@ -1600,8 +1620,7 @@ def test_pickling(self): self.assertEqual(orig, derived) self.assertEqual(orig.__reduce__(), orig.__reduce_ex__(2)) - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_compat_unpickle(self): tests = [ b"cdatetime\ndate\n(S'\\x07\\xdf\\x0b\\x1b'\ntR.", @@ -1703,29 +1722,44 @@ def test_replace(self): cls = self.theclass args = [1, 2, 3] base = cls(*args) - self.assertEqual(base, base.replace()) + self.assertEqual(base.replace(), base) + self.assertEqual(copy.replace(base), base) - i = 0 - for name, newval in (("year", 2), - ("month", 3), - ("day", 4)): + changes = (("year", 2), + ("month", 3), + ("day", 4)) + for i, (name, newval) in enumerate(changes): newargs = args[:] newargs[i] = newval expected = cls(*newargs) - got = base.replace(**{name: newval}) - self.assertEqual(expected, got) - i += 1 + self.assertEqual(base.replace(**{name: newval}), expected) + self.assertEqual(copy.replace(base, **{name: newval}), expected) # Out of bounds. base = cls(2000, 2, 29) self.assertRaises(ValueError, base.replace, year=2001) + self.assertRaises(ValueError, copy.replace, base, year=2001) def test_subclass_replace(self): class DateSubclass(self.theclass): - pass + def __new__(cls, *args, **kwargs): + result = self.theclass.__new__(cls, *args, **kwargs) + result.extra = 7 + return result dt = DateSubclass(2012, 1, 1) - self.assertIs(type(dt.replace(year=2013)), DateSubclass) + + test_cases = [ + ('self.replace', dt.replace(year=2013)), + ('copy.replace', copy.replace(dt, year=2013)), + ] + + for name, res in test_cases: + with self.subTest(name): + self.assertIs(type(res), DateSubclass) + self.assertEqual(res.year, 2013) + self.assertEqual(res.month, 1) + self.assertEqual(res.extra, 7) def test_subclass_date(self): @@ -1912,6 +1946,10 @@ def test_fromisoformat_fails(self): '2009-02-29', # Invalid leap day '2019-W53-1', # No week 53 in 2019 '2020-W54-1', # No week 54 + '0000-W25-1', # Invalid year + '10000-W25-1', # Invalid year + '2020-W25-0', # Invalid day-of-week + '2020-W25-8', # Invalid day-of-week '2009\ud80002\ud80028', # Separators are surrogate codepoints ] @@ -2369,8 +2407,7 @@ def test_pickling_subclass_datetime(self): self.assertEqual(orig, derived) self.assertTrue(isinstance(derived, SubclassDatetime)) - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_compat_unpickle(self): tests = [ b'cdatetime\ndatetime\n(' @@ -2780,6 +2817,20 @@ def test_strptime_single_digit(self): newdate = strptime(string, format) self.assertEqual(newdate, target, msg=reason) + @warnings_helper.ignore_warnings(category=DeprecationWarning) + def test_strptime_leap_year(self): + # GH-70647: warns if parsing a format with a day and no year. + with self.assertRaises(ValueError): + # The existing behavior that GH-70647 seeks to change. + self.theclass.strptime('02-29', '%m-%d') + with self.assertWarnsRegex(DeprecationWarning, + r'.*day of month without a year.*'): + self.theclass.strptime('03-14.159265', '%m-%d.%f') + with self._assertNotWarns(DeprecationWarning): + self.theclass.strptime('20-03-14.159265', '%y-%m-%d.%f') + with self._assertNotWarns(DeprecationWarning): + self.theclass.strptime('02-29,2024', '%m-%d,%Y') + def test_more_timetuple(self): # This tests fields beyond those tested by the TestDate.test_timetuple. t = self.theclass(2004, 12, 31, 6, 22, 33) @@ -2813,11 +2864,34 @@ def test_more_strftime(self): self.assertEqual(t.strftime("%z"), "-0200" + z) self.assertEqual(t.strftime("%:z"), "-02:00:" + z) - # bpo-34482: Check that surrogates don't cause a crash. - try: - t.strftime('%y\ud800%m %H\ud800%M') - except UnicodeEncodeError: - pass + @unittest.skip('TODO: RUSTPYTHON') + # UnicodeEncodeError: 'utf-8' codec can't encode character '\ud83d' in position 0: surrogates not allowed + def test_strftime_special(self): + t = self.theclass(2004, 12, 31, 6, 22, 33, 47) + s1 = t.strftime('%c') + s2 = t.strftime('%B') + # gh-52551, gh-78662: Unicode strings should pass through strftime, + # independently from locale. + self.assertEqual(t.strftime('\U0001f40d'), '\U0001f40d') + self.assertEqual(t.strftime('\U0001f4bb%c\U0001f40d%B'), f'\U0001f4bb{s1}\U0001f40d{s2}') + self.assertEqual(t.strftime('%c\U0001f4bb%B\U0001f40d'), f'{s1}\U0001f4bb{s2}\U0001f40d') + # Lone surrogates should pass through. + self.assertEqual(t.strftime('\ud83d'), '\ud83d') + self.assertEqual(t.strftime('\udc0d'), '\udc0d') + self.assertEqual(t.strftime('\ud83d%c\udc0d%B'), f'\ud83d{s1}\udc0d{s2}') + self.assertEqual(t.strftime('%c\ud83d%B\udc0d'), f'{s1}\ud83d{s2}\udc0d') + self.assertEqual(t.strftime('%c\udc0d%B\ud83d'), f'{s1}\udc0d{s2}\ud83d') + # Surrogate pairs should not recombine. + self.assertEqual(t.strftime('\ud83d\udc0d'), '\ud83d\udc0d') + self.assertEqual(t.strftime('%c\ud83d\udc0d%B'), f'{s1}\ud83d\udc0d{s2}') + # Surrogate-escaped bytes should not recombine. + self.assertEqual(t.strftime('\udcf0\udc9f\udc90\udc8d'), '\udcf0\udc9f\udc90\udc8d') + self.assertEqual(t.strftime('%c\udcf0\udc9f\udc90\udc8d%B'), f'{s1}\udcf0\udc9f\udc90\udc8d{s2}') + # gh-124531: The null character should not terminate the format string. + self.assertEqual(t.strftime('\0'), '\0') + self.assertEqual(t.strftime('\0'*1000), '\0'*1000) + self.assertEqual(t.strftime('\0%c\0%B'), f'\0{s1}\0{s2}') + self.assertEqual(t.strftime('%c\0%B\0'), f'{s1}\0{s2}\0') def test_extract(self): dt = self.theclass(2002, 3, 4, 18, 45, 3, 1234) @@ -2862,26 +2936,27 @@ def test_replace(self): cls = self.theclass args = [1, 2, 3, 4, 5, 6, 7] base = cls(*args) - self.assertEqual(base, base.replace()) - - i = 0 - for name, newval in (("year", 2), - ("month", 3), - ("day", 4), - ("hour", 5), - ("minute", 6), - ("second", 7), - ("microsecond", 8)): + self.assertEqual(base.replace(), base) + self.assertEqual(copy.replace(base), base) + + changes = (("year", 2), + ("month", 3), + ("day", 4), + ("hour", 5), + ("minute", 6), + ("second", 7), + ("microsecond", 8)) + for i, (name, newval) in enumerate(changes): newargs = args[:] newargs[i] = newval expected = cls(*newargs) - got = base.replace(**{name: newval}) - self.assertEqual(expected, got) - i += 1 + self.assertEqual(base.replace(**{name: newval}), expected) + self.assertEqual(copy.replace(base, **{name: newval}), expected) # Out of bounds. base = cls(2000, 2, 29) self.assertRaises(ValueError, base.replace, year=2001) + self.assertRaises(ValueError, copy.replace, base, year=2001) @support.run_with_tz('EDT4') def test_astimezone(self): @@ -3024,6 +3099,26 @@ def __new__(cls, *args, **kwargs): self.assertIsInstance(dt, DateTimeSubclass) self.assertEqual(dt.extra, 7) + def test_subclass_replace_fold(self): + class DateTimeSubclass(self.theclass): + pass + + dt = DateTimeSubclass(2012, 1, 1) + dt2 = DateTimeSubclass(2012, 1, 1, fold=1) + + test_cases = [ + ('self.replace', dt.replace(year=2013), 0), + ('self.replace', dt2.replace(year=2013), 1), + ('copy.replace', copy.replace(dt, year=2013), 0), + ('copy.replace', copy.replace(dt2, year=2013), 1), + ] + + for name, res, fold in test_cases: + with self.subTest(name, fold=fold): + self.assertIs(type(res), DateTimeSubclass) + self.assertEqual(res.year, 2013) + self.assertEqual(res.fold, fold) + def test_fromisoformat_datetime(self): # Test that isoformat() is reversible base_dates = [ @@ -3301,6 +3396,9 @@ def test_fromisoformat_fails_datetime(self): '2009-04-19T12:30:45.123456-05:00a', # Extra text '2009-04-19T12:30:45.123-05:00a', # Extra text '2009-04-19T12:30:45-05:00a', # Extra text + '2009-04-19T12:30:45.400 +02:30', # Space between ms and timezone (gh-130959) + '2009-04-19T12:30:45.400 ', # Trailing space (gh-130959) + '2009-04-19T12:30:45. 400', # Space before fraction (gh-130959) ] for bad_str in bad_strs: @@ -3570,6 +3668,35 @@ def test_strftime(self): # gh-85432: The parameter was named "fmt" in the pure-Python impl. t.strftime(format="%f") + @unittest.skip('TODO: RUSTPYTHON') + # UnicodeEncodeError: 'utf-8' codec can't encode character '\ud83d' in position 0: surrogates not allowed + def test_strftime_special(self): + t = self.theclass(1, 2, 3, 4) + s1 = t.strftime('%I%p%Z') + s2 = t.strftime('%X') + # gh-52551, gh-78662: Unicode strings should pass through strftime, + # independently from locale. + self.assertEqual(t.strftime('\U0001f40d'), '\U0001f40d') + self.assertEqual(t.strftime('\U0001f4bb%I%p%Z\U0001f40d%X'), f'\U0001f4bb{s1}\U0001f40d{s2}') + self.assertEqual(t.strftime('%I%p%Z\U0001f4bb%X\U0001f40d'), f'{s1}\U0001f4bb{s2}\U0001f40d') + # Lone surrogates should pass through. + self.assertEqual(t.strftime('\ud83d'), '\ud83d') + self.assertEqual(t.strftime('\udc0d'), '\udc0d') + self.assertEqual(t.strftime('\ud83d%I%p%Z\udc0d%X'), f'\ud83d{s1}\udc0d{s2}') + self.assertEqual(t.strftime('%I%p%Z\ud83d%X\udc0d'), f'{s1}\ud83d{s2}\udc0d') + self.assertEqual(t.strftime('%I%p%Z\udc0d%X\ud83d'), f'{s1}\udc0d{s2}\ud83d') + # Surrogate pairs should not recombine. + self.assertEqual(t.strftime('\ud83d\udc0d'), '\ud83d\udc0d') + self.assertEqual(t.strftime('%I%p%Z\ud83d\udc0d%X'), f'{s1}\ud83d\udc0d{s2}') + # Surrogate-escaped bytes should not recombine. + self.assertEqual(t.strftime('\udcf0\udc9f\udc90\udc8d'), '\udcf0\udc9f\udc90\udc8d') + self.assertEqual(t.strftime('%I%p%Z\udcf0\udc9f\udc90\udc8d%X'), f'{s1}\udcf0\udc9f\udc90\udc8d{s2}') + # gh-124531: The null character should not terminate the format string. + self.assertEqual(t.strftime('\0'), '\0') + self.assertEqual(t.strftime('\0'*1000), '\0'*1000) + self.assertEqual(t.strftime('\0%I%p%Z\0%X'), f'\0{s1}\0{s2}') + self.assertEqual(t.strftime('%I%p%Z\0%X\0'), f'{s1}\0{s2}\0') + def test_format(self): t = self.theclass(1, 2, 3, 4) self.assertEqual(t.__format__(''), str(t)) @@ -3641,8 +3768,7 @@ def test_pickling_subclass_time(self): self.assertEqual(orig, derived) self.assertTrue(isinstance(derived, SubclassTime)) - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_compat_unpickle(self): tests = [ (b"cdatetime\ntime\n(S'\\x14;\\x10\\x00\\x10\\x00'\ntR.", @@ -3679,19 +3805,19 @@ def test_replace(self): cls = self.theclass args = [1, 2, 3, 4] base = cls(*args) - self.assertEqual(base, base.replace()) - - i = 0 - for name, newval in (("hour", 5), - ("minute", 6), - ("second", 7), - ("microsecond", 8)): + self.assertEqual(base.replace(), base) + self.assertEqual(copy.replace(base), base) + + changes = (("hour", 5), + ("minute", 6), + ("second", 7), + ("microsecond", 8)) + for i, (name, newval) in enumerate(changes): newargs = args[:] newargs[i] = newval expected = cls(*newargs) - got = base.replace(**{name: newval}) - self.assertEqual(expected, got) - i += 1 + self.assertEqual(base.replace(**{name: newval}), expected) + self.assertEqual(copy.replace(base, **{name: newval}), expected) # Out of bounds. base = cls(1) @@ -3699,13 +3825,35 @@ def test_replace(self): self.assertRaises(ValueError, base.replace, minute=-1) self.assertRaises(ValueError, base.replace, second=100) self.assertRaises(ValueError, base.replace, microsecond=1000000) + self.assertRaises(ValueError, copy.replace, base, hour=24) + self.assertRaises(ValueError, copy.replace, base, minute=-1) + self.assertRaises(ValueError, copy.replace, base, second=100) + self.assertRaises(ValueError, copy.replace, base, microsecond=1000000) def test_subclass_replace(self): class TimeSubclass(self.theclass): - pass + def __new__(cls, *args, **kwargs): + result = self.theclass.__new__(cls, *args, **kwargs) + result.extra = 7 + return result ctime = TimeSubclass(12, 30) - self.assertIs(type(ctime.replace(hour=10)), TimeSubclass) + ctime2 = TimeSubclass(12, 30, fold=1) + + test_cases = [ + ('self.replace', ctime.replace(hour=10), 0), + ('self.replace', ctime2.replace(hour=10), 1), + ('copy.replace', copy.replace(ctime, hour=10), 0), + ('copy.replace', copy.replace(ctime2, hour=10), 1), + ] + + for name, res, fold in test_cases: + with self.subTest(name, fold=fold): + self.assertIs(type(res), TimeSubclass) + self.assertEqual(res.hour, 10) + self.assertEqual(res.minute, 30) + self.assertEqual(res.extra, 7) + self.assertEqual(res.fold, fold) def test_subclass_time(self): @@ -3918,6 +4066,8 @@ def test_empty(self): self.assertEqual(t.microsecond, 0) self.assertIsNone(t.tzinfo) + @unittest.skip('TODO: RUSTPYTHON') + # UnicodeEncodeError: 'utf-8' codec can't encode character '\ud800' in position 0: surrogates not allowed def test_zones(self): est = FixedOffset(-300, "EST", 1) utc = FixedOffset(0, "UTC", -2) @@ -4001,9 +4151,8 @@ def tzname(self, dt): return self.tz self.assertRaises(TypeError, t.strftime, "%Z") # Issue #6697: - if '_Fast' in self.__class__.__name__: - Badtzname.tz = '\ud800' - self.assertRaises(ValueError, t.strftime, "%Z") + Badtzname.tz = '\ud800' + self.assertEqual(t.strftime("%Z"), '\ud800') def test_hash_edge_cases(self): # Offsets that overflow a basic time. @@ -4037,8 +4186,7 @@ def test_pickling(self): self.assertEqual(derived.tzname(), 'cookie') self.assertEqual(orig.__reduce__(), orig.__reduce_ex__(2)) - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_compat_unpickle(self): tests = [ b"cdatetime\ntime\n(S'\\x05\\x06\\x07\\x01\\xe2@'\n" @@ -4095,31 +4243,37 @@ def test_replace(self): zm200 = FixedOffset(timedelta(minutes=-200), "-200") args = [1, 2, 3, 4, z100] base = cls(*args) - self.assertEqual(base, base.replace()) - - i = 0 - for name, newval in (("hour", 5), - ("minute", 6), - ("second", 7), - ("microsecond", 8), - ("tzinfo", zm200)): + self.assertEqual(base.replace(), base) + self.assertEqual(copy.replace(base), base) + + changes = (("hour", 5), + ("minute", 6), + ("second", 7), + ("microsecond", 8), + ("tzinfo", zm200)) + for i, (name, newval) in enumerate(changes): newargs = args[:] newargs[i] = newval expected = cls(*newargs) - got = base.replace(**{name: newval}) - self.assertEqual(expected, got) - i += 1 + self.assertEqual(base.replace(**{name: newval}), expected) + self.assertEqual(copy.replace(base, **{name: newval}), expected) # Ensure we can get rid of a tzinfo. self.assertEqual(base.tzname(), "+100") base2 = base.replace(tzinfo=None) self.assertIsNone(base2.tzinfo) self.assertIsNone(base2.tzname()) + base22 = copy.replace(base, tzinfo=None) + self.assertIsNone(base22.tzinfo) + self.assertIsNone(base22.tzname()) # Ensure we can add one. base3 = base2.replace(tzinfo=z100) self.assertEqual(base, base3) self.assertIs(base.tzinfo, base3.tzinfo) + base32 = copy.replace(base22, tzinfo=z100) + self.assertEqual(base, base32) + self.assertIs(base.tzinfo, base32.tzinfo) # Out of bounds. base = cls(1) @@ -4127,6 +4281,10 @@ def test_replace(self): self.assertRaises(ValueError, base.replace, minute=-1) self.assertRaises(ValueError, base.replace, second=100) self.assertRaises(ValueError, base.replace, microsecond=1000000) + self.assertRaises(ValueError, copy.replace, base, hour=24) + self.assertRaises(ValueError, copy.replace, base, minute=-1) + self.assertRaises(ValueError, copy.replace, base, second=100) + self.assertRaises(ValueError, copy.replace, base, microsecond=1000000) def test_mixed_compare(self): t1 = self.theclass(1, 2, 3) @@ -4331,6 +4489,9 @@ def test_fromisoformat_fails(self): '12:30:45.123456-', # Extra at end of microsecond time '12:30:45.123456+', # Extra at end of microsecond time '12:30:45.123456+12:00:30a', # Extra at end of full time + '12:30:45.400 +02:30', # Space between ms and timezone (gh-130959) + '12:30:45.400 ', # Trailing space (gh-130959) + '12:30:45. 400', # Space before fraction (gh-130959) ] for bad_str in bad_strs: @@ -4491,8 +4652,7 @@ def test_pickling(self): self.assertEqual(derived.tzname(), 'cookie') self.assertEqual(orig.__reduce__(), orig.__reduce_ex__(2)) - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_compat_unpickle(self): tests = [ b'cdatetime\ndatetime\n' @@ -4897,38 +5057,45 @@ def test_replace(self): zm200 = FixedOffset(timedelta(minutes=-200), "-200") args = [1, 2, 3, 4, 5, 6, 7, z100] base = cls(*args) - self.assertEqual(base, base.replace()) - - i = 0 - for name, newval in (("year", 2), - ("month", 3), - ("day", 4), - ("hour", 5), - ("minute", 6), - ("second", 7), - ("microsecond", 8), - ("tzinfo", zm200)): + self.assertEqual(base.replace(), base) + self.assertEqual(copy.replace(base), base) + + changes = (("year", 2), + ("month", 3), + ("day", 4), + ("hour", 5), + ("minute", 6), + ("second", 7), + ("microsecond", 8), + ("tzinfo", zm200)) + for i, (name, newval) in enumerate(changes): newargs = args[:] newargs[i] = newval expected = cls(*newargs) - got = base.replace(**{name: newval}) - self.assertEqual(expected, got) - i += 1 + self.assertEqual(base.replace(**{name: newval}), expected) + self.assertEqual(copy.replace(base, **{name: newval}), expected) # Ensure we can get rid of a tzinfo. self.assertEqual(base.tzname(), "+100") base2 = base.replace(tzinfo=None) self.assertIsNone(base2.tzinfo) self.assertIsNone(base2.tzname()) + base22 = copy.replace(base, tzinfo=None) + self.assertIsNone(base22.tzinfo) + self.assertIsNone(base22.tzname()) # Ensure we can add one. base3 = base2.replace(tzinfo=z100) self.assertEqual(base, base3) self.assertIs(base.tzinfo, base3.tzinfo) + base32 = copy.replace(base22, tzinfo=z100) + self.assertEqual(base, base32) + self.assertIs(base.tzinfo, base32.tzinfo) # Out of bounds. base = cls(2000, 2, 29) self.assertRaises(ValueError, base.replace, year=2001) + self.assertRaises(ValueError, copy.replace, base, year=2001) def test_more_astimezone(self): # The inherited test_astimezone covered some trivial and error cases. @@ -5418,42 +5585,50 @@ def fromutc(self, dt): class Oddballs(unittest.TestCase): - def test_bug_1028306(self): + def test_date_datetime_comparison(self): + # bpo-1028306, bpo-5516 (gh-49766) # Trying to compare a date to a datetime should act like a mixed- # type comparison, despite that datetime is a subclass of date. as_date = date.today() as_datetime = datetime.combine(as_date, time()) - self.assertTrue(as_date != as_datetime) - self.assertTrue(as_datetime != as_date) - self.assertFalse(as_date == as_datetime) - self.assertFalse(as_datetime == as_date) - self.assertRaises(TypeError, lambda: as_date < as_datetime) - self.assertRaises(TypeError, lambda: as_datetime < as_date) - self.assertRaises(TypeError, lambda: as_date <= as_datetime) - self.assertRaises(TypeError, lambda: as_datetime <= as_date) - self.assertRaises(TypeError, lambda: as_date > as_datetime) - self.assertRaises(TypeError, lambda: as_datetime > as_date) - self.assertRaises(TypeError, lambda: as_date >= as_datetime) - self.assertRaises(TypeError, lambda: as_datetime >= as_date) - - # Nevertheless, comparison should work with the base-class (date) - # projection if use of a date method is forced. - self.assertEqual(as_date.__eq__(as_datetime), True) - different_day = (as_date.day + 1) % 20 + 1 - as_different = as_datetime.replace(day= different_day) - self.assertEqual(as_date.__eq__(as_different), False) + date_sc = SubclassDate(as_date.year, as_date.month, as_date.day) + datetime_sc = SubclassDatetime(as_date.year, as_date.month, + as_date.day, 0, 0, 0) + for d in (as_date, date_sc): + for dt in (as_datetime, datetime_sc): + for x, y in (d, dt), (dt, d): + self.assertTrue(x != y) + self.assertFalse(x == y) + self.assertRaises(TypeError, lambda: x < y) + self.assertRaises(TypeError, lambda: x <= y) + self.assertRaises(TypeError, lambda: x > y) + self.assertRaises(TypeError, lambda: x >= y) # And date should compare with other subclasses of date. If a # subclass wants to stop this, it's up to the subclass to do so. - date_sc = SubclassDate(as_date.year, as_date.month, as_date.day) - self.assertEqual(as_date, date_sc) - self.assertEqual(date_sc, as_date) - # Ditto for datetimes. - datetime_sc = SubclassDatetime(as_datetime.year, as_datetime.month, - as_date.day, 0, 0, 0) - self.assertEqual(as_datetime, datetime_sc) - self.assertEqual(datetime_sc, as_datetime) + for x, y in ((as_date, date_sc), + (date_sc, as_date), + (as_datetime, datetime_sc), + (datetime_sc, as_datetime)): + self.assertTrue(x == y) + self.assertFalse(x != y) + self.assertFalse(x < y) + self.assertFalse(x > y) + self.assertTrue(x <= y) + self.assertTrue(x >= y) + + # Nevertheless, comparison should work if other object is an instance + # of date or datetime class with overridden comparison operators. + # So special methods should return NotImplemented, as if + # date and datetime were independent classes. + for x, y in (as_date, as_datetime), (as_datetime, as_date): + self.assertEqual(x.__eq__(y), NotImplemented) + self.assertEqual(x.__ne__(y), NotImplemented) + self.assertEqual(x.__lt__(y), NotImplemented) + self.assertEqual(x.__gt__(y), NotImplemented) + self.assertEqual(x.__gt__(y), NotImplemented) + self.assertEqual(x.__ge__(y), NotImplemented) def test_extra_attributes(self): with self.assertWarns(DeprecationWarning): @@ -6679,6 +6854,126 @@ def test_datetime_from_timestamp(self): self.assertEqual(dt_orig, dt_rt) + def test_type_check_in_subinterp(self): + # iOS requires the use of the custom framework loader, + # not the ExtensionFileLoader. + if sys.platform == "ios": + extension_loader = "AppleFrameworkLoader" + else: + extension_loader = "ExtensionFileLoader" + + script = textwrap.dedent(f""" + if {_interpreters is None}: + import _testcapi as module + module.test_datetime_capi() + else: + import importlib.machinery + import importlib.util + fullname = '_testcapi_datetime' + origin = importlib.util.find_spec('_testcapi').origin + loader = importlib.machinery.{extension_loader}(fullname, origin) + spec = importlib.util.spec_from_loader(fullname, loader) + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + + def run(type_checker, obj): + if not type_checker(obj, True): + raise TypeError(f'{{type(obj)}} is not C API type') + + import _datetime + run(module.datetime_check_date, _datetime.date.today()) + run(module.datetime_check_datetime, _datetime.datetime.now()) + run(module.datetime_check_time, _datetime.time(12, 30)) + run(module.datetime_check_delta, _datetime.timedelta(1)) + run(module.datetime_check_tzinfo, _datetime.tzinfo()) + """) + if _interpreters is None: + ret = support.run_in_subinterp(script) + self.assertEqual(ret, 0) + else: + for name in ('isolated', 'legacy'): + with self.subTest(name): + config = _interpreters.new_config(name).__dict__ + ret = support.run_in_subinterp_with_config(script, **config) + self.assertEqual(ret, 0) + + +class ExtensionModuleTests(unittest.TestCase): + + def setUp(self): + if self.__class__.__name__.endswith('Pure'): + self.skipTest('Not relevant in pure Python') + + @support.cpython_only + def test_gh_120161(self): + with self.subTest('simple'): + script = textwrap.dedent(""" + import datetime + from _ast import Tuple + f = lambda: None + Tuple.dims = property(f, f) + + class tzutc(datetime.tzinfo): + pass + """) + script_helper.assert_python_ok('-c', script) + + with self.subTest('complex'): + script = textwrap.dedent(""" + import asyncio + import datetime + from typing import Type + + class tzutc(datetime.tzinfo): + pass + _EPOCHTZ = datetime.datetime(1970, 1, 1, tzinfo=tzutc()) + + class FakeDateMeta(type): + def __instancecheck__(self, obj): + return True + class FakeDate(datetime.date, metaclass=FakeDateMeta): + pass + def pickle_fake_date(datetime_) -> Type[FakeDate]: + # A pickle function for FakeDate + return FakeDate + """) + script_helper.assert_python_ok('-c', script) + + # TODO: RUSTPYTHON + # AssertionError: Process return code is 1 + @unittest.expectedFailure + def test_update_type_cache(self): + # gh-120782 + script = textwrap.dedent(""" + import sys + for i in range(5): + import _datetime + assert _datetime.date.max > _datetime.date.min + assert _datetime.time.max > _datetime.time.min + assert _datetime.datetime.max > _datetime.datetime.min + assert _datetime.timedelta.max > _datetime.timedelta.min + assert _datetime.date.__dict__["min"] is _datetime.date.min + assert _datetime.date.__dict__["max"] is _datetime.date.max + assert _datetime.date.__dict__["resolution"] is _datetime.date.resolution + assert _datetime.time.__dict__["min"] is _datetime.time.min + assert _datetime.time.__dict__["max"] is _datetime.time.max + assert _datetime.time.__dict__["resolution"] is _datetime.time.resolution + assert _datetime.datetime.__dict__["min"] is _datetime.datetime.min + assert _datetime.datetime.__dict__["max"] is _datetime.datetime.max + assert _datetime.datetime.__dict__["resolution"] is _datetime.datetime.resolution + assert _datetime.timedelta.__dict__["min"] is _datetime.timedelta.min + assert _datetime.timedelta.__dict__["max"] is _datetime.timedelta.max + assert _datetime.timedelta.__dict__["resolution"] is _datetime.timedelta.resolution + assert _datetime.timezone.__dict__["min"] is _datetime.timezone.min + assert _datetime.timezone.__dict__["max"] is _datetime.timezone.max + assert _datetime.timezone.__dict__["utc"] is _datetime.timezone.utc + assert isinstance(_datetime.timezone.min, _datetime.tzinfo) + assert isinstance(_datetime.timezone.max, _datetime.tzinfo) + assert isinstance(_datetime.timezone.utc, _datetime.tzinfo) + del sys.modules['_datetime'] + """) + script_helper.assert_python_ok('-c', script) + def load_tests(loader, standard_tests, pattern): standard_tests.addTest(ZoneInfoCompleteTest()) diff --git a/Lib/test/test_code_module.py b/Lib/test/test_code_module.py index 346cf6746a7..4d5463e6db3 100644 --- a/Lib/test/test_code_module.py +++ b/Lib/test/test_code_module.py @@ -64,6 +64,7 @@ def test_console_stderr(self): raise AssertionError("no console stdout") # TODO: RUSTPYTHON + # AssertionError: Lists differ: [' F[27 chars] x = ?', ' ^', 'SyntaxError: got unexpected token ?'] != [' F[27 chars] x = ?', ' ^', 'SyntaxError: invalid syntax'] @unittest.expectedFailure def test_syntax_error(self): self.infunc.side_effect = ["def f():", @@ -86,6 +87,7 @@ def test_syntax_error(self): self.assertIs(self.sysmod.last_exc, self.sysmod.last_value) # TODO: RUSTPYTHON + # AssertionError: Lists differ: [' F[15 chars], line 1', ' 1', 'IndentationError: unexpected indentation'] != [' F[15 chars], line 1', ' 1', 'IndentationError: unexpected indent'] @unittest.expectedFailure def test_indentation_error(self): self.infunc.side_effect = [" 1", EOFError('Finished')] @@ -104,6 +106,7 @@ def test_indentation_error(self): self.assertIs(self.sysmod.last_exc, self.sysmod.last_value) # TODO: RUSTPYTHON + # AssertionError: False is not true : UnicodeDecodeError: invalid utf-8 sequence of 1 bytes from index 1 @unittest.expectedFailure def test_unicode_error(self): self.infunc.side_effect = ["'\ud800'", EOFError('Finished')] @@ -142,6 +145,7 @@ def test_sysexcepthook(self): 'ValueError: BOOM!\n']) # TODO: RUSTPYTHON + # AssertionError: Lists differ: [' F[35 chars]= ?\n', ' ^\n', 'SyntaxError: got unexpected token ?\n'] != [' F[35 chars]= ?\n', ' ^\n', 'SyntaxError: invalid syntax\n'] @unittest.expectedFailure def test_sysexcepthook_syntax_error(self): self.infunc.side_effect = ["def f():", @@ -167,6 +171,7 @@ def test_sysexcepthook_syntax_error(self): 'SyntaxError: invalid syntax\n']) # TODO: RUSTPYTHON + # AssertionError: Lists differ: [' F[21 chars] 1\n', ' 1\n', 'IndentationError: unexpected indentation\n'] != [' F[21 chars] 1\n', ' 1\n', 'IndentationError: unexpected indent\n'] @unittest.expectedFailure def test_sysexcepthook_indentation_error(self): self.infunc.side_effect = [" 1", EOFError('Finished')] @@ -262,8 +267,7 @@ def test_exit_msg(self): self.assertEqual(err_msg, ['write', (expected,), {}]) - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_cause_tb(self): self.infunc.side_effect = ["raise ValueError('') from AttributeError", EOFError('Finished')] From cd012f9b2e5bea893d7ba483fe34c5189ef2fd17 Mon Sep 17 00:00:00 2001 From: Terry Tianlin Luan <44815845+terryluan12@users.noreply.github.com> Date: Tue, 30 Dec 2025 19:43:10 -0500 Subject: [PATCH 0013/1575] Added doctest tests (#6598) --- Lib/test/test_doctest/__init__.py | 5 + Lib/test/test_doctest/decorator_mod.py | 10 + Lib/test/test_doctest/doctest_aliases.py | 13 + Lib/test/test_doctest/doctest_lineno.py | 107 + Lib/test/test_doctest/sample_doctest.py | 76 + .../test_doctest/sample_doctest_errors.py | 46 + .../sample_doctest_no_docstrings.py | 12 + .../sample_doctest_no_doctests.py | 15 + Lib/test/test_doctest/sample_doctest_skip.py | 49 + Lib/test/test_doctest/test_doctest.py | 4021 +++++++++++++++++ Lib/test/test_doctest/test_doctest.txt | 17 + Lib/test/test_doctest/test_doctest2.py | 126 + Lib/test/test_doctest/test_doctest2.txt | 14 + Lib/test/test_doctest/test_doctest3.txt | 5 + Lib/test/test_doctest/test_doctest4.txt | 11 + Lib/test/test_doctest/test_doctest_errors.txt | 14 + Lib/test/test_doctest/test_doctest_skip.txt | 6 + Lib/test/test_doctest/test_doctest_skip2.txt | 6 + crates/vm/src/builtins/int.rs | 2 +- 19 files changed, 4554 insertions(+), 1 deletion(-) create mode 100644 Lib/test/test_doctest/__init__.py create mode 100644 Lib/test/test_doctest/decorator_mod.py create mode 100644 Lib/test/test_doctest/doctest_aliases.py create mode 100644 Lib/test/test_doctest/doctest_lineno.py create mode 100644 Lib/test/test_doctest/sample_doctest.py create mode 100644 Lib/test/test_doctest/sample_doctest_errors.py create mode 100644 Lib/test/test_doctest/sample_doctest_no_docstrings.py create mode 100644 Lib/test/test_doctest/sample_doctest_no_doctests.py create mode 100644 Lib/test/test_doctest/sample_doctest_skip.py create mode 100644 Lib/test/test_doctest/test_doctest.py create mode 100644 Lib/test/test_doctest/test_doctest.txt create mode 100644 Lib/test/test_doctest/test_doctest2.py create mode 100644 Lib/test/test_doctest/test_doctest2.txt create mode 100644 Lib/test/test_doctest/test_doctest3.txt create mode 100644 Lib/test/test_doctest/test_doctest4.txt create mode 100644 Lib/test/test_doctest/test_doctest_errors.txt create mode 100644 Lib/test/test_doctest/test_doctest_skip.txt create mode 100644 Lib/test/test_doctest/test_doctest_skip2.txt diff --git a/Lib/test/test_doctest/__init__.py b/Lib/test/test_doctest/__init__.py new file mode 100644 index 00000000000..4b16ecc3115 --- /dev/null +++ b/Lib/test/test_doctest/__init__.py @@ -0,0 +1,5 @@ +import os +from test.support import load_package_tests + +def load_tests(*args): + return load_package_tests(os.path.dirname(__file__), *args) diff --git a/Lib/test/test_doctest/decorator_mod.py b/Lib/test/test_doctest/decorator_mod.py new file mode 100644 index 00000000000..9f106888411 --- /dev/null +++ b/Lib/test/test_doctest/decorator_mod.py @@ -0,0 +1,10 @@ +# This module is used in `doctest_lineno.py`. +import functools + + +def decorator(f): + @functools.wraps(f) + def inner(): + return f() + + return inner diff --git a/Lib/test/test_doctest/doctest_aliases.py b/Lib/test/test_doctest/doctest_aliases.py new file mode 100644 index 00000000000..30cefafa83e --- /dev/null +++ b/Lib/test/test_doctest/doctest_aliases.py @@ -0,0 +1,13 @@ +# Used by test_doctest.py. + +class TwoNames: + '''f() and g() are two names for the same method''' + + def f(self): + ''' + >>> print(TwoNames().f()) + f + ''' + return 'f' + + g = f # define an alias for f diff --git a/Lib/test/test_doctest/doctest_lineno.py b/Lib/test/test_doctest/doctest_lineno.py new file mode 100644 index 00000000000..0bd402e9828 --- /dev/null +++ b/Lib/test/test_doctest/doctest_lineno.py @@ -0,0 +1,107 @@ +# This module is used in `test_doctest`. +# It must not have a docstring. + +def func_with_docstring(): + """Some unrelated info.""" + + +def func_without_docstring(): + pass + + +def func_with_doctest(): + """ + This function really contains a test case. + + >>> func_with_doctest.__name__ + 'func_with_doctest' + """ + return 3 + + +class ClassWithDocstring: + """Some unrelated class information.""" + + +class ClassWithoutDocstring: + pass + + +class ClassWithDoctest: + """This class really has a test case in it. + + >>> ClassWithDoctest.__name__ + 'ClassWithDoctest' + """ + + +class MethodWrapper: + def method_with_docstring(self): + """Method with a docstring.""" + + def method_without_docstring(self): + pass + + def method_with_doctest(self): + """ + This has a doctest! + >>> MethodWrapper.method_with_doctest.__name__ + 'method_with_doctest' + """ + + @classmethod + def classmethod_with_doctest(cls): + """ + This has a doctest! + >>> MethodWrapper.classmethod_with_doctest.__name__ + 'classmethod_with_doctest' + """ + + @property + def property_with_doctest(self): + """ + This has a doctest! + >>> MethodWrapper.property_with_doctest.__name__ + 'property_with_doctest' + """ + +# https://github.com/python/cpython/issues/99433 +str_wrapper = object().__str__ + + +# https://github.com/python/cpython/issues/115392 +from test.test_doctest.decorator_mod import decorator + +@decorator +@decorator +def func_with_docstring_wrapped(): + """Some unrelated info.""" + + +# https://github.com/python/cpython/issues/136914 +import functools + + +@functools.cache +def cached_func_with_doctest(value): + """ + >>> cached_func_with_doctest(1) + -1 + """ + return -value + + +@functools.cache +def cached_func_without_docstring(value): + return value + 1 + + +class ClassWithACachedProperty: + + @functools.cached_property + def cached(self): + """ + >>> X().cached + -1 + """ + return 0 diff --git a/Lib/test/test_doctest/sample_doctest.py b/Lib/test/test_doctest/sample_doctest.py new file mode 100644 index 00000000000..049f737a0a4 --- /dev/null +++ b/Lib/test/test_doctest/sample_doctest.py @@ -0,0 +1,76 @@ +"""This is a sample module that doesn't really test anything all that + interesting. + +It simply has a few tests, some of which succeed and some of which fail. + +It's important that the numbers remain constant as another test is +testing the running of these tests. + + +>>> 2+2 +4 +""" + + +def foo(): + """ + + >>> 2+2 + 5 + + >>> 2+2 + 4 + """ + +def bar(): + """ + + >>> 2+2 + 4 + """ + +def test_silly_setup(): + """ + + >>> import test.test_doctest.test_doctest + >>> test.test_doctest.test_doctest.sillySetup + True + """ + +def w_blank(): + """ + >>> if 1: + ... print('a') + ... print() + ... print('b') + a + + b + """ + +x = 1 +def x_is_one(): + """ + >>> x + 1 + """ + +def y_is_one(): + """ + >>> y + 1 + """ + +__test__ = {'good': """ + >>> 42 + 42 + """, + 'bad': """ + >>> 42 + 666 + """, + } + +def test_suite(): + import doctest + return doctest.DocTestSuite() diff --git a/Lib/test/test_doctest/sample_doctest_errors.py b/Lib/test/test_doctest/sample_doctest_errors.py new file mode 100644 index 00000000000..4a6f07af2d4 --- /dev/null +++ b/Lib/test/test_doctest/sample_doctest_errors.py @@ -0,0 +1,46 @@ +"""This is a sample module used for testing doctest. + +This module includes various scenarios involving errors. + +>>> 2 + 2 +5 +>>> 1/0 +1 +""" + +def g(): + [][0] # line 12 + +def errors(): + """ + >>> 2 + 2 + 5 + >>> 1/0 + 1 + >>> def f(): + ... 2 + '2' + ... + >>> f() + 1 + >>> g() + 1 + """ + +def syntax_error(): + """ + >>> 2+*3 + 5 + """ + +__test__ = { + 'bad': """ + >>> 2 + 2 + 5 + >>> 1/0 + 1 + """, +} + +def test_suite(): + import doctest + return doctest.DocTestSuite() diff --git a/Lib/test/test_doctest/sample_doctest_no_docstrings.py b/Lib/test/test_doctest/sample_doctest_no_docstrings.py new file mode 100644 index 00000000000..e4201edbce9 --- /dev/null +++ b/Lib/test/test_doctest/sample_doctest_no_docstrings.py @@ -0,0 +1,12 @@ +# This is a sample module used for testing doctest. +# +# This module is for testing how doctest handles a module with no +# docstrings. + + +class Foo(object): + + # A class with no docstring. + + def __init__(self): + pass diff --git a/Lib/test/test_doctest/sample_doctest_no_doctests.py b/Lib/test/test_doctest/sample_doctest_no_doctests.py new file mode 100644 index 00000000000..7daa57231c8 --- /dev/null +++ b/Lib/test/test_doctest/sample_doctest_no_doctests.py @@ -0,0 +1,15 @@ +"""This is a sample module used for testing doctest. + +This module is for testing how doctest handles a module with docstrings +but no doctest examples. + +""" + + +class Foo(object): + """A docstring with no doctest examples. + + """ + + def __init__(self): + pass diff --git a/Lib/test/test_doctest/sample_doctest_skip.py b/Lib/test/test_doctest/sample_doctest_skip.py new file mode 100644 index 00000000000..1b83dec1f8c --- /dev/null +++ b/Lib/test/test_doctest/sample_doctest_skip.py @@ -0,0 +1,49 @@ +"""This is a sample module used for testing doctest. + +This module includes various scenarios involving skips. +""" + +def no_skip_pass(): + """ + >>> 2 + 2 + 4 + """ + +def no_skip_fail(): + """ + >>> 2 + 2 + 5 + """ + +def single_skip(): + """ + >>> 2 + 2 # doctest: +SKIP + 4 + """ + +def double_skip(): + """ + >>> 2 + 2 # doctest: +SKIP + 4 + >>> 3 + 3 # doctest: +SKIP + 6 + """ + +def partial_skip_pass(): + """ + >>> 2 + 2 # doctest: +SKIP + 4 + >>> 3 + 3 + 6 + """ + +def partial_skip_fail(): + """ + >>> 2 + 2 # doctest: +SKIP + 4 + >>> 2 + 2 + 5 + """ + +def no_examples(): + """A docstring with no examples should not be counted as run or skipped.""" diff --git a/Lib/test/test_doctest/test_doctest.py b/Lib/test/test_doctest/test_doctest.py new file mode 100644 index 00000000000..48d5ff6f73f --- /dev/null +++ b/Lib/test/test_doctest/test_doctest.py @@ -0,0 +1,4021 @@ +""" +Test script for doctest. +""" + +from test import support +from test.support import import_helper +from test.support.pty_helper import FakeInput # used in doctests +import doctest +import functools +import os +import sys +import importlib +import importlib.abc +import importlib.util +import unittest +import tempfile +import types +import contextlib +import _colorize + + +def doctest_skip_if(condition): + def decorator(func): + if condition and support.HAVE_DOCSTRINGS: + func.__doc__ = ">>> pass # doctest: +SKIP" + return func + return decorator + + +# NOTE: There are some additional tests relating to interaction with +# zipimport in the test_zipimport_support test module. +# There are also related tests in `test_doctest2` module. + +###################################################################### +## Sample Objects (used by test cases) +###################################################################### + +def sample_func(v): + """ + Blah blah + + >>> print(sample_func(22)) + 44 + + Yee ha! + """ + return v+v + +class SampleClass: + """ + >>> print(1) + 1 + + >>> # comments get ignored. so are empty PS1 and PS2 prompts: + >>> + ... + + Multiline example: + >>> sc = SampleClass(3) + >>> for i in range(10): + ... sc = sc.double() + ... print(' ', sc.get(), sep='', end='') + 6 12 24 48 96 192 384 768 1536 3072 + """ + def __init__(self, val): + """ + >>> print(SampleClass(12).get()) + 12 + """ + self.val = val + + def double(self): + """ + >>> print(SampleClass(12).double().get()) + 24 + """ + return SampleClass(self.val + self.val) + + def get(self): + """ + >>> print(SampleClass(-5).get()) + -5 + """ + return self.val + + def setter(self, val): + """ + >>> s = SampleClass(-5) + >>> s.setter(1) + >>> print(s.val) + 1 + """ + self.val = val + + def a_staticmethod(v): + """ + >>> print(SampleClass.a_staticmethod(10)) + 11 + """ + return v+1 + a_staticmethod = staticmethod(a_staticmethod) + + def a_classmethod(cls, v): + """ + >>> print(SampleClass.a_classmethod(10)) + 12 + >>> print(SampleClass(0).a_classmethod(10)) + 12 + """ + return v+2 + a_classmethod = classmethod(a_classmethod) + + a_property = property(get, setter, doc=""" + >>> print(SampleClass(22).a_property) + 22 + """) + + a_class_attribute = 42 + + @functools.cached_property + def a_cached_property(self): + """ + >>> print(SampleClass(29).get()) + 29 + """ + return "hello" + + class NestedClass: + """ + >>> x = SampleClass.NestedClass(5) + >>> y = x.square() + >>> print(y.get()) + 25 + """ + def __init__(self, val=0): + """ + >>> print(SampleClass.NestedClass().get()) + 0 + """ + self.val = val + def square(self): + return SampleClass.NestedClass(self.val*self.val) + def get(self): + return self.val + +class SampleNewStyleClass(object): + r""" + >>> print('1\n2\n3') + 1 + 2 + 3 + """ + def __init__(self, val): + """ + >>> print(SampleNewStyleClass(12).get()) + 12 + """ + self.val = val + + def double(self): + """ + >>> print(SampleNewStyleClass(12).double().get()) + 24 + """ + return SampleNewStyleClass(self.val + self.val) + + def get(self): + """ + >>> print(SampleNewStyleClass(-5).get()) + -5 + """ + return self.val + +###################################################################### +## Test Cases +###################################################################### + +def test_Example(): r""" +Unit tests for the `Example` class. + +Example is a simple container class that holds: + - `source`: A source string. + - `want`: An expected output string. + - `exc_msg`: An expected exception message string (or None if no + exception is expected). + - `lineno`: A line number (within the docstring). + - `indent`: The example's indentation in the input string. + - `options`: An option dictionary, mapping option flags to True or + False. + +These attributes are set by the constructor. `source` and `want` are +required; the other attributes all have default values: + + >>> example = doctest.Example('print(1)', '1\n') + >>> (example.source, example.want, example.exc_msg, + ... example.lineno, example.indent, example.options) + ('print(1)\n', '1\n', None, 0, 0, {}) + +The first three attributes (`source`, `want`, and `exc_msg`) may be +specified positionally; the remaining arguments should be specified as +keyword arguments: + + >>> exc_msg = 'IndexError: pop from an empty list' + >>> example = doctest.Example('[].pop()', '', exc_msg, + ... lineno=5, indent=4, + ... options={doctest.ELLIPSIS: True}) + >>> (example.source, example.want, example.exc_msg, + ... example.lineno, example.indent, example.options) + ('[].pop()\n', '', 'IndexError: pop from an empty list\n', 5, 4, {8: True}) + +The constructor normalizes the `source` string to end in a newline: + + Source spans a single line: no terminating newline. + >>> e = doctest.Example('print(1)', '1\n') + >>> e.source, e.want + ('print(1)\n', '1\n') + + >>> e = doctest.Example('print(1)\n', '1\n') + >>> e.source, e.want + ('print(1)\n', '1\n') + + Source spans multiple lines: require terminating newline. + >>> e = doctest.Example('print(1);\nprint(2)\n', '1\n2\n') + >>> e.source, e.want + ('print(1);\nprint(2)\n', '1\n2\n') + + >>> e = doctest.Example('print(1);\nprint(2)', '1\n2\n') + >>> e.source, e.want + ('print(1);\nprint(2)\n', '1\n2\n') + + Empty source string (which should never appear in real examples) + >>> e = doctest.Example('', '') + >>> e.source, e.want + ('\n', '') + +The constructor normalizes the `want` string to end in a newline, +unless it's the empty string: + + >>> e = doctest.Example('print(1)', '1\n') + >>> e.source, e.want + ('print(1)\n', '1\n') + + >>> e = doctest.Example('print(1)', '1') + >>> e.source, e.want + ('print(1)\n', '1\n') + + >>> e = doctest.Example('print', '') + >>> e.source, e.want + ('print\n', '') + +The constructor normalizes the `exc_msg` string to end in a newline, +unless it's `None`: + + Message spans one line + >>> exc_msg = 'IndexError: pop from an empty list' + >>> e = doctest.Example('[].pop()', '', exc_msg) + >>> e.exc_msg + 'IndexError: pop from an empty list\n' + + >>> exc_msg = 'IndexError: pop from an empty list\n' + >>> e = doctest.Example('[].pop()', '', exc_msg) + >>> e.exc_msg + 'IndexError: pop from an empty list\n' + + Message spans multiple lines + >>> exc_msg = 'ValueError: 1\n 2' + >>> e = doctest.Example('raise ValueError("1\n 2")', '', exc_msg) + >>> e.exc_msg + 'ValueError: 1\n 2\n' + + >>> exc_msg = 'ValueError: 1\n 2\n' + >>> e = doctest.Example('raise ValueError("1\n 2")', '', exc_msg) + >>> e.exc_msg + 'ValueError: 1\n 2\n' + + Empty (but non-None) exception message (which should never appear + in real examples) + >>> exc_msg = '' + >>> e = doctest.Example('raise X()', '', exc_msg) + >>> e.exc_msg + '\n' + +Compare `Example`: + >>> example = doctest.Example('print 1', '1\n') + >>> same_example = doctest.Example('print 1', '1\n') + >>> other_example = doctest.Example('print 42', '42\n') + >>> example == same_example + True + >>> example != same_example + False + >>> hash(example) == hash(same_example) + True + >>> example == other_example + False + >>> example != other_example + True +""" + +def test_DocTest(): r""" +Unit tests for the `DocTest` class. + +DocTest is a collection of examples, extracted from a docstring, along +with information about where the docstring comes from (a name, +filename, and line number). The docstring is parsed by the `DocTest` +constructor: + + >>> docstring = ''' + ... >>> print(12) + ... 12 + ... + ... Non-example text. + ... + ... >>> print('another\\example') + ... another + ... example + ... ''' + >>> globs = {} # globals to run the test in. + >>> parser = doctest.DocTestParser() + >>> test = parser.get_doctest(docstring, globs, 'some_test', + ... 'some_file', 20) + >>> print(test) + + >>> len(test.examples) + 2 + >>> e1, e2 = test.examples + >>> (e1.source, e1.want, e1.lineno) + ('print(12)\n', '12\n', 1) + >>> (e2.source, e2.want, e2.lineno) + ("print('another\\example')\n", 'another\nexample\n', 6) + +Source information (name, filename, and line number) is available as +attributes on the doctest object: + + >>> (test.name, test.filename, test.lineno) + ('some_test', 'some_file', 20) + +The line number of an example within its containing file is found by +adding the line number of the example and the line number of its +containing test: + + >>> test.lineno + e1.lineno + 21 + >>> test.lineno + e2.lineno + 26 + +If the docstring contains inconsistent leading whitespace in the +expected output of an example, then `DocTest` will raise a ValueError: + + >>> docstring = r''' + ... >>> print('bad\nindentation') + ... bad + ... indentation + ... ''' + >>> parser.get_doctest(docstring, globs, 'some_test', 'filename', 0) + Traceback (most recent call last): + ValueError: line 4 of the docstring for some_test has inconsistent leading whitespace: 'indentation' + +If the docstring contains inconsistent leading whitespace on +continuation lines, then `DocTest` will raise a ValueError: + + >>> docstring = r''' + ... >>> print(('bad indentation', + ... ... 2)) + ... ('bad', 'indentation') + ... ''' + >>> parser.get_doctest(docstring, globs, 'some_test', 'filename', 0) + Traceback (most recent call last): + ValueError: line 2 of the docstring for some_test has inconsistent leading whitespace: '... 2))' + +If there's no blank space after a PS1 prompt ('>>>'), then `DocTest` +will raise a ValueError: + + >>> docstring = '>>>print(1)\n1' + >>> parser.get_doctest(docstring, globs, 'some_test', 'filename', 0) + Traceback (most recent call last): + ValueError: line 1 of the docstring for some_test lacks blank after >>>: '>>>print(1)' + +If there's no blank space after a PS2 prompt ('...'), then `DocTest` +will raise a ValueError: + + >>> docstring = '>>> if 1:\n...print(1)\n1' + >>> parser.get_doctest(docstring, globs, 'some_test', 'filename', 0) + Traceback (most recent call last): + ValueError: line 2 of the docstring for some_test lacks blank after ...: '...print(1)' + +Compare `DocTest`: + + >>> docstring = ''' + ... >>> print 12 + ... 12 + ... ''' + >>> test = parser.get_doctest(docstring, globs, 'some_test', + ... 'some_test', 20) + >>> same_test = parser.get_doctest(docstring, globs, 'some_test', + ... 'some_test', 20) + >>> test == same_test + True + >>> test != same_test + False + >>> hash(test) == hash(same_test) + True + >>> docstring = ''' + ... >>> print 42 + ... 42 + ... ''' + >>> other_test = parser.get_doctest(docstring, globs, 'other_test', + ... 'other_file', 10) + >>> test == other_test + False + >>> test != other_test + True + >>> test < other_test + False + >>> other_test < test + True + +Test comparison with lineno None on one side + + >>> no_lineno = parser.get_doctest(docstring, globs, 'some_test', + ... 'some_test', None) + >>> test.lineno is None + False + >>> no_lineno.lineno is None + True + >>> test < no_lineno + False + >>> no_lineno < test + True + +Compare `DocTestCase`: + + >>> DocTestCase = doctest.DocTestCase + >>> test_case = DocTestCase(test) + >>> same_test_case = DocTestCase(same_test) + >>> other_test_case = DocTestCase(other_test) + >>> test_case == same_test_case + True + >>> test_case != same_test_case + False + >>> hash(test_case) == hash(same_test_case) + True + >>> test == other_test_case + False + >>> test != other_test_case + True + +""" + +class test_DocTestFinder: + def basics(): r""" +Unit tests for the `DocTestFinder` class. + +DocTestFinder is used to extract DocTests from an object's docstring +and the docstrings of its contained objects. It can be used with +modules, functions, classes, methods, staticmethods, classmethods, and +properties. + +Finding Tests in Functions +~~~~~~~~~~~~~~~~~~~~~~~~~~ +For a function whose docstring contains examples, DocTestFinder.find() +will return a single test (for that function's docstring): + + >>> finder = doctest.DocTestFinder() + +We'll simulate a __file__ attr that ends in pyc: + + >>> from test.test_doctest import test_doctest + >>> old = test_doctest.__file__ + >>> test_doctest.__file__ = 'test_doctest.pyc' + + >>> tests = finder.find(sample_func) + + >>> print(tests) # doctest: +ELLIPSIS + [] + +The exact name depends on how test_doctest was invoked, so allow for +leading path components. + + >>> tests[0].filename # doctest: +ELLIPSIS + '...test_doctest.py' + + >>> test_doctest.__file__ = old + + + >>> e = tests[0].examples[0] + >>> (e.source, e.want, e.lineno) + ('print(sample_func(22))\n', '44\n', 3) + +By default, tests are created for objects with no docstring: + + >>> def no_docstring(v): + ... pass + >>> finder.find(no_docstring) + [] + +However, the optional argument `exclude_empty` to the DocTestFinder +constructor can be used to exclude tests for objects with empty +docstrings: + + >>> def no_docstring(v): + ... pass + >>> excl_empty_finder = doctest.DocTestFinder(exclude_empty=True) + >>> excl_empty_finder.find(no_docstring) + [] + +If the function has a docstring with no examples, then a test with no +examples is returned. (This lets `DocTestRunner` collect statistics +about which functions have no tests -- but is that useful? And should +an empty test also be created when there's no docstring?) + + >>> def no_examples(v): + ... ''' no doctest examples ''' + >>> finder.find(no_examples) # doctest: +ELLIPSIS + [] + +Finding Tests in Classes +~~~~~~~~~~~~~~~~~~~~~~~~ +For a class, DocTestFinder will create a test for the class's +docstring, and will recursively explore its contents, including +methods, classmethods, staticmethods, properties, and nested classes. + + >>> finder = doctest.DocTestFinder() + >>> tests = finder.find(SampleClass) + >>> for t in tests: + ... print('%2s %s' % (len(t.examples), t.name)) + 3 SampleClass + 3 SampleClass.NestedClass + 1 SampleClass.NestedClass.__init__ + 1 SampleClass.__init__ + 1 SampleClass.a_cached_property + 2 SampleClass.a_classmethod + 1 SampleClass.a_property + 1 SampleClass.a_staticmethod + 1 SampleClass.double + 1 SampleClass.get + 3 SampleClass.setter + +New-style classes are also supported: + + >>> tests = finder.find(SampleNewStyleClass) + >>> for t in tests: + ... print('%2s %s' % (len(t.examples), t.name)) + 1 SampleNewStyleClass + 1 SampleNewStyleClass.__init__ + 1 SampleNewStyleClass.double + 1 SampleNewStyleClass.get + +Finding Tests in Modules +~~~~~~~~~~~~~~~~~~~~~~~~ +For a module, DocTestFinder will create a test for the class's +docstring, and will recursively explore its contents, including +functions, classes, and the `__test__` dictionary, if it exists: + + >>> # A module + >>> import types + >>> m = types.ModuleType('some_module') + >>> def triple(val): + ... ''' + ... >>> print(triple(11)) + ... 33 + ... ''' + ... return val*3 + >>> m.__dict__.update({ + ... 'sample_func': sample_func, + ... 'SampleClass': SampleClass, + ... '__doc__': ''' + ... Module docstring. + ... >>> print('module') + ... module + ... ''', + ... '__test__': { + ... 'd': '>>> print(6)\n6\n>>> print(7)\n7\n', + ... 'c': triple}}) + + >>> finder = doctest.DocTestFinder() + >>> # Use module=test_doctest, to prevent doctest from + >>> # ignoring the objects since they weren't defined in m. + >>> from test.test_doctest import test_doctest + >>> tests = finder.find(m, module=test_doctest) + >>> for t in tests: + ... print('%2s %s' % (len(t.examples), t.name)) + 1 some_module + 3 some_module.SampleClass + 3 some_module.SampleClass.NestedClass + 1 some_module.SampleClass.NestedClass.__init__ + 1 some_module.SampleClass.__init__ + 1 some_module.SampleClass.a_cached_property + 2 some_module.SampleClass.a_classmethod + 1 some_module.SampleClass.a_property + 1 some_module.SampleClass.a_staticmethod + 1 some_module.SampleClass.double + 1 some_module.SampleClass.get + 3 some_module.SampleClass.setter + 1 some_module.__test__.c + 2 some_module.__test__.d + 1 some_module.sample_func + +However, doctest will ignore imported objects from other modules +(without proper `module=`): + + >>> import types + >>> m = types.ModuleType('poluted_namespace') + >>> m.__dict__.update({ + ... 'sample_func': sample_func, + ... 'SampleClass': SampleClass, + ... }) + + >>> finder = doctest.DocTestFinder() + >>> finder.find(m) + [] + +Duplicate Removal +~~~~~~~~~~~~~~~~~ +If a single object is listed twice (under different names), then tests +will only be generated for it once: + + >>> from test.test_doctest import doctest_aliases + >>> assert doctest_aliases.TwoNames.f + >>> assert doctest_aliases.TwoNames.g + >>> tests = excl_empty_finder.find(doctest_aliases) + >>> print(len(tests)) + 2 + >>> print(tests[0].name) + test.test_doctest.doctest_aliases.TwoNames + + TwoNames.f and TwoNames.g are bound to the same object. + We can't guess which will be found in doctest's traversal of + TwoNames.__dict__ first, so we have to allow for either. + + >>> tests[1].name.split('.')[-1] in ['f', 'g'] + True + +Empty Tests +~~~~~~~~~~~ +By default, an object with no doctests doesn't create any tests: + + >>> tests = doctest.DocTestFinder().find(SampleClass) + >>> for t in tests: + ... print('%2s %s' % (len(t.examples), t.name)) + 3 SampleClass + 3 SampleClass.NestedClass + 1 SampleClass.NestedClass.__init__ + 1 SampleClass.__init__ + 1 SampleClass.a_cached_property + 2 SampleClass.a_classmethod + 1 SampleClass.a_property + 1 SampleClass.a_staticmethod + 1 SampleClass.double + 1 SampleClass.get + 3 SampleClass.setter + +By default, that excluded objects with no doctests. exclude_empty=False +tells it to include (empty) tests for objects with no doctests. This feature +is really to support backward compatibility in what doctest.master.summarize() +displays. + + >>> tests = doctest.DocTestFinder(exclude_empty=False).find(SampleClass) + >>> for t in tests: + ... print('%2s %s' % (len(t.examples), t.name)) + 3 SampleClass + 3 SampleClass.NestedClass + 1 SampleClass.NestedClass.__init__ + 0 SampleClass.NestedClass.get + 0 SampleClass.NestedClass.square + 1 SampleClass.__init__ + 1 SampleClass.a_cached_property + 2 SampleClass.a_classmethod + 1 SampleClass.a_property + 1 SampleClass.a_staticmethod + 1 SampleClass.double + 1 SampleClass.get + 3 SampleClass.setter + +When used with `exclude_empty=False` we are also interested in line numbers +of doctests that are empty. +It used to be broken for quite some time until `bpo-28249`. + + >>> from test.test_doctest import doctest_lineno + >>> tests = doctest.DocTestFinder(exclude_empty=False).find(doctest_lineno) + >>> for t in tests: + ... print('%5s %s' % (t.lineno, t.name)) + None test.test_doctest.doctest_lineno + None test.test_doctest.doctest_lineno.ClassWithACachedProperty + 102 test.test_doctest.doctest_lineno.ClassWithACachedProperty.cached + 22 test.test_doctest.doctest_lineno.ClassWithDocstring + 30 test.test_doctest.doctest_lineno.ClassWithDoctest + None test.test_doctest.doctest_lineno.ClassWithoutDocstring + None test.test_doctest.doctest_lineno.MethodWrapper + 53 test.test_doctest.doctest_lineno.MethodWrapper.classmethod_with_doctest + 39 test.test_doctest.doctest_lineno.MethodWrapper.method_with_docstring + 45 test.test_doctest.doctest_lineno.MethodWrapper.method_with_doctest + None test.test_doctest.doctest_lineno.MethodWrapper.method_without_docstring + 61 test.test_doctest.doctest_lineno.MethodWrapper.property_with_doctest + 86 test.test_doctest.doctest_lineno.cached_func_with_doctest + None test.test_doctest.doctest_lineno.cached_func_without_docstring + 4 test.test_doctest.doctest_lineno.func_with_docstring + 77 test.test_doctest.doctest_lineno.func_with_docstring_wrapped + 12 test.test_doctest.doctest_lineno.func_with_doctest + None test.test_doctest.doctest_lineno.func_without_docstring + +Turning off Recursion +~~~~~~~~~~~~~~~~~~~~~ +DocTestFinder can be told not to look for tests in contained objects +using the `recurse` flag: + + >>> tests = doctest.DocTestFinder(recurse=False).find(SampleClass) + >>> for t in tests: + ... print('%2s %s' % (len(t.examples), t.name)) + 3 SampleClass + +Line numbers +~~~~~~~~~~~~ +DocTestFinder finds the line number of each example: + + >>> def f(x): + ... ''' + ... >>> x = 12 + ... + ... some text + ... + ... >>> # examples are not created for comments & bare prompts. + ... >>> + ... ... + ... + ... >>> for x in range(10): + ... ... print(x, end=' ') + ... 0 1 2 3 4 5 6 7 8 9 + ... >>> x//2 + ... 6 + ... ''' + >>> test = doctest.DocTestFinder().find(f)[0] + >>> [e.lineno for e in test.examples] + [1, 9, 12] +""" + +# TODO: RUSTPYTHON +# Currently, only 5 builtins types exist in RustPython +# So this test will fail +# if int.__doc__: # simple check for --without-doc-strings, skip if lacking +# def non_Python_modules(): r""" + +# Finding Doctests in Modules Not Written in Python +# ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +# DocTestFinder can also find doctests in most modules not written in Python. +# We'll use builtins as an example, since it almost certainly isn't written in +# plain ol' Python and is guaranteed to be available. + +# >>> import builtins +# >>> tests = doctest.DocTestFinder().find(builtins) +# >>> 830 < len(tests) < 860 # approximate number of objects with docstrings +# True +# >>> real_tests = [t for t in tests if len(t.examples) > 0] +# >>> len(real_tests) # objects that actually have doctests +# 14 +# >>> for t in real_tests: +# ... print('{} {}'.format(len(t.examples), t.name)) +# ... +# 1 builtins.bin +# 5 builtins.bytearray.hex +# 5 builtins.bytes.hex +# 3 builtins.float.as_integer_ratio +# 2 builtins.float.fromhex +# 2 builtins.float.hex +# 1 builtins.hex +# 1 builtins.int +# 3 builtins.int.as_integer_ratio +# 2 builtins.int.bit_count +# 2 builtins.int.bit_length +# 5 builtins.memoryview.hex +# 1 builtins.oct +# 1 builtins.zip + +# Note here that 'bin', 'oct', and 'hex' are functions; 'float.as_integer_ratio', +# 'float.hex', and 'int.bit_length' are methods; 'float.fromhex' is a classmethod, +# and 'int' is a type. +# """ + + +class TestDocTest(unittest.TestCase): + + def test_run(self): + test = ''' + >>> 1 + 1 + 11 + >>> 2 + 3 # doctest: +SKIP + "23" + >>> 5 + 7 + 57 + ''' + + def myfunc(): + pass + myfunc.__doc__ = test + + # test DocTestFinder.run() + test = doctest.DocTestFinder().find(myfunc)[0] + with support.captured_stdout(): + with support.captured_stderr(): + results = doctest.DocTestRunner(verbose=False).run(test) + + # test TestResults + self.assertIsInstance(results, doctest.TestResults) + self.assertEqual(results.failed, 2) + self.assertEqual(results.attempted, 3) + self.assertEqual(results.skipped, 1) + self.assertEqual(tuple(results), (2, 3)) + x, y = results + self.assertEqual((x, y), (2, 3)) + + +class TestDocTestFinder(unittest.TestCase): + + def test_issue35753(self): + # This import of `call` should trigger issue35753 when + # DocTestFinder.find() is called due to inspect.unwrap() failing, + # however with a patched doctest this should succeed. + from unittest.mock import call + dummy_module = types.ModuleType("dummy") + dummy_module.__dict__['inject_call'] = call + finder = doctest.DocTestFinder() + self.assertEqual(finder.find(dummy_module), []) + + def test_empty_namespace_package(self): + pkg_name = 'doctest_empty_pkg' + with tempfile.TemporaryDirectory() as parent_dir: + pkg_dir = os.path.join(parent_dir, pkg_name) + os.mkdir(pkg_dir) + sys.path.append(parent_dir) + try: + mod = importlib.import_module(pkg_name) + finally: + import_helper.forget(pkg_name) + sys.path.pop() + + include_empty_finder = doctest.DocTestFinder(exclude_empty=False) + exclude_empty_finder = doctest.DocTestFinder(exclude_empty=True) + + self.assertEqual(len(include_empty_finder.find(mod)), 1) + self.assertEqual(len(exclude_empty_finder.find(mod)), 0) + +def test_DocTestParser(): r""" +Unit tests for the `DocTestParser` class. + +DocTestParser is used to parse docstrings containing doctest examples. + +The `parse` method divides a docstring into examples and intervening +text: + + >>> s = ''' + ... >>> x, y = 2, 3 # no output expected + ... >>> if 1: + ... ... print(x) + ... ... print(y) + ... 2 + ... 3 + ... + ... Some text. + ... >>> x+y + ... 5 + ... ''' + >>> parser = doctest.DocTestParser() + >>> for piece in parser.parse(s): + ... if isinstance(piece, doctest.Example): + ... print('Example:', (piece.source, piece.want, piece.lineno)) + ... else: + ... print(' Text:', repr(piece)) + Text: '\n' + Example: ('x, y = 2, 3 # no output expected\n', '', 1) + Text: '' + Example: ('if 1:\n print(x)\n print(y)\n', '2\n3\n', 2) + Text: '\nSome text.\n' + Example: ('x+y\n', '5\n', 9) + Text: '' + +The `get_examples` method returns just the examples: + + >>> for piece in parser.get_examples(s): + ... print((piece.source, piece.want, piece.lineno)) + ('x, y = 2, 3 # no output expected\n', '', 1) + ('if 1:\n print(x)\n print(y)\n', '2\n3\n', 2) + ('x+y\n', '5\n', 9) + +The `get_doctest` method creates a Test from the examples, along with the +given arguments: + + >>> test = parser.get_doctest(s, {}, 'name', 'filename', lineno=5) + >>> (test.name, test.filename, test.lineno) + ('name', 'filename', 5) + >>> for piece in test.examples: + ... print((piece.source, piece.want, piece.lineno)) + ('x, y = 2, 3 # no output expected\n', '', 1) + ('if 1:\n print(x)\n print(y)\n', '2\n3\n', 2) + ('x+y\n', '5\n', 9) +""" + +class test_DocTestRunner: + def basics(): r""" +Unit tests for the `DocTestRunner` class. + +DocTestRunner is used to run DocTest test cases, and to accumulate +statistics. Here's a simple DocTest case we can use: + + >>> save_colorize = _colorize.COLORIZE + >>> _colorize.COLORIZE = False + + >>> def f(x): + ... ''' + ... >>> x = 12 + ... >>> print(x) + ... 12 + ... >>> x//2 + ... 6 + ... ''' + >>> test = doctest.DocTestFinder().find(f)[0] + +The main DocTestRunner interface is the `run` method, which runs a +given DocTest case in a given namespace (globs). It returns a tuple +`(f,t)`, where `f` is the number of failed tests and `t` is the number +of tried tests. + + >>> doctest.DocTestRunner(verbose=False).run(test) + TestResults(failed=0, attempted=3) + +If any example produces incorrect output, then the test runner reports +the failure and proceeds to the next example: + + >>> def f(x): + ... ''' + ... >>> x = 12 + ... >>> print(x) + ... 14 + ... >>> x//2 + ... 6 + ... ''' + >>> test = doctest.DocTestFinder().find(f)[0] + >>> doctest.DocTestRunner(verbose=True).run(test) + ... # doctest: +ELLIPSIS + Trying: + x = 12 + Expecting nothing + ok + Trying: + print(x) + Expecting: + 14 + ********************************************************************** + File ..., line 4, in f + Failed example: + print(x) + Expected: + 14 + Got: + 12 + Trying: + x//2 + Expecting: + 6 + ok + TestResults(failed=1, attempted=3) + + >>> _colorize.COLORIZE = save_colorize +""" + def verbose_flag(): r""" +The `verbose` flag makes the test runner generate more detailed +output: + + >>> def f(x): + ... ''' + ... >>> x = 12 + ... >>> print(x) + ... 12 + ... >>> x//2 + ... 6 + ... ''' + >>> test = doctest.DocTestFinder().find(f)[0] + + >>> doctest.DocTestRunner(verbose=True).run(test) + Trying: + x = 12 + Expecting nothing + ok + Trying: + print(x) + Expecting: + 12 + ok + Trying: + x//2 + Expecting: + 6 + ok + TestResults(failed=0, attempted=3) + +If the `verbose` flag is unspecified, then the output will be verbose +iff `-v` appears in sys.argv: + + >>> # Save the real sys.argv list. + >>> old_argv = sys.argv + + >>> # If -v does not appear in sys.argv, then output isn't verbose. + >>> sys.argv = ['test'] + >>> doctest.DocTestRunner().run(test) + TestResults(failed=0, attempted=3) + + >>> # If -v does appear in sys.argv, then output is verbose. + >>> sys.argv = ['test', '-v'] + >>> doctest.DocTestRunner().run(test) + Trying: + x = 12 + Expecting nothing + ok + Trying: + print(x) + Expecting: + 12 + ok + Trying: + x//2 + Expecting: + 6 + ok + TestResults(failed=0, attempted=3) + + >>> # Restore sys.argv + >>> sys.argv = old_argv + +In the remaining examples, the test runner's verbosity will be +explicitly set, to ensure that the test behavior is consistent. + """ + def exceptions(): r""" +Tests of `DocTestRunner`'s exception handling. + +An expected exception is specified with a traceback message. The +lines between the first line and the type/value may be omitted or +replaced with any other string: + + >>> save_colorize = _colorize.COLORIZE + >>> _colorize.COLORIZE = False + + >>> def f(x): + ... ''' + ... >>> x = 12 + ... >>> print(x//0) + ... Traceback (most recent call last): + ... ZeroDivisionError: integer division or modulo by zero + ... ''' + >>> test = doctest.DocTestFinder().find(f)[0] + >>> doctest.DocTestRunner(verbose=False).run(test) + TestResults(failed=0, attempted=2) + +An example may not generate output before it raises an exception; if +it does, then the traceback message will not be recognized as +signaling an expected exception, so the example will be reported as an +unexpected exception: + + >>> def f(x): + ... ''' + ... >>> x = 12 + ... >>> print('pre-exception output', x//0) + ... pre-exception output + ... Traceback (most recent call last): + ... ZeroDivisionError: integer division or modulo by zero + ... ''' + >>> test = doctest.DocTestFinder().find(f)[0] + >>> doctest.DocTestRunner(verbose=False).run(test) + ... # doctest: +ELLIPSIS + ********************************************************************** + File ..., line 4, in f + Failed example: + print('pre-exception output', x//0) + Exception raised: + ... + ZeroDivisionError: integer division or modulo by zero + TestResults(failed=1, attempted=2) + +Exception messages may contain newlines: + + >>> def f(x): + ... r''' + ... >>> raise ValueError('multi\nline\nmessage') + ... Traceback (most recent call last): + ... ValueError: multi + ... line + ... message + ... ''' + >>> test = doctest.DocTestFinder().find(f)[0] + >>> doctest.DocTestRunner(verbose=False).run(test) + TestResults(failed=0, attempted=1) + +If an exception is expected, but an exception with the wrong type or +message is raised, then it is reported as a failure: + + >>> def f(x): + ... r''' + ... >>> raise ValueError('message') + ... Traceback (most recent call last): + ... ValueError: wrong message + ... ''' + >>> test = doctest.DocTestFinder().find(f)[0] + >>> doctest.DocTestRunner(verbose=False).run(test) + ... # doctest: +ELLIPSIS + ********************************************************************** + File ..., line 3, in f + Failed example: + raise ValueError('message') + Expected: + Traceback (most recent call last): + ValueError: wrong message + Got: + Traceback (most recent call last): + ... + ValueError: message + TestResults(failed=1, attempted=1) + +However, IGNORE_EXCEPTION_DETAIL can be used to allow a mismatch in the +detail: + + >>> def f(x): + ... r''' + ... >>> raise ValueError('message') #doctest: +IGNORE_EXCEPTION_DETAIL + ... Traceback (most recent call last): + ... ValueError: wrong message + ... ''' + >>> test = doctest.DocTestFinder().find(f)[0] + >>> doctest.DocTestRunner(verbose=False).run(test) + TestResults(failed=0, attempted=1) + +IGNORE_EXCEPTION_DETAIL also ignores difference in exception formatting +between Python versions. For example, in Python 2.x, the module path of +the exception is not in the output, but this will fail under Python 3: + + >>> def f(x): + ... r''' + ... >>> from http.client import HTTPException + ... >>> raise HTTPException('message') + ... Traceback (most recent call last): + ... HTTPException: message + ... ''' + >>> test = doctest.DocTestFinder().find(f)[0] + >>> doctest.DocTestRunner(verbose=False).run(test) + ... # doctest: +ELLIPSIS + ********************************************************************** + File ..., line 4, in f + Failed example: + raise HTTPException('message') + Expected: + Traceback (most recent call last): + HTTPException: message + Got: + Traceback (most recent call last): + ... + http.client.HTTPException: message + TestResults(failed=1, attempted=2) + +But in Python 3 the module path is included, and therefore a test must look +like the following test to succeed in Python 3. But that test will fail under +Python 2. + + >>> def f(x): + ... r''' + ... >>> from http.client import HTTPException + ... >>> raise HTTPException('message') + ... Traceback (most recent call last): + ... http.client.HTTPException: message + ... ''' + >>> test = doctest.DocTestFinder().find(f)[0] + >>> doctest.DocTestRunner(verbose=False).run(test) + TestResults(failed=0, attempted=2) + +However, with IGNORE_EXCEPTION_DETAIL, the module name of the exception +(or its unexpected absence) will be ignored: + + >>> def f(x): + ... r''' + ... >>> from http.client import HTTPException + ... >>> raise HTTPException('message') #doctest: +IGNORE_EXCEPTION_DETAIL + ... Traceback (most recent call last): + ... HTTPException: message + ... ''' + >>> test = doctest.DocTestFinder().find(f)[0] + >>> doctest.DocTestRunner(verbose=False).run(test) + TestResults(failed=0, attempted=2) + +The module path will be completely ignored, so two different module paths will +still pass if IGNORE_EXCEPTION_DETAIL is given. This is intentional, so it can +be used when exceptions have changed module. + + >>> def f(x): + ... r''' + ... >>> from http.client import HTTPException + ... >>> raise HTTPException('message') #doctest: +IGNORE_EXCEPTION_DETAIL + ... Traceback (most recent call last): + ... foo.bar.HTTPException: message + ... ''' + >>> test = doctest.DocTestFinder().find(f)[0] + >>> doctest.DocTestRunner(verbose=False).run(test) + TestResults(failed=0, attempted=2) + +But IGNORE_EXCEPTION_DETAIL does not allow a mismatch in the exception type: + + >>> def f(x): + ... r''' + ... >>> raise ValueError('message') #doctest: +IGNORE_EXCEPTION_DETAIL + ... Traceback (most recent call last): + ... TypeError: wrong type + ... ''' + >>> test = doctest.DocTestFinder().find(f)[0] + >>> doctest.DocTestRunner(verbose=False).run(test) + ... # doctest: +ELLIPSIS + ********************************************************************** + File ..., line 3, in f + Failed example: + raise ValueError('message') #doctest: +IGNORE_EXCEPTION_DETAIL + Expected: + Traceback (most recent call last): + TypeError: wrong type + Got: + Traceback (most recent call last): + ... + ValueError: message + TestResults(failed=1, attempted=1) + +If the exception does not have a message, you can still use +IGNORE_EXCEPTION_DETAIL to normalize the modules between Python 2 and 3: + + >>> def f(x): + ... r''' + ... >>> from http.client import HTTPException + ... >>> raise HTTPException() #doctest: +IGNORE_EXCEPTION_DETAIL + ... Traceback (most recent call last): + ... foo.bar.HTTPException + ... ''' + >>> test = doctest.DocTestFinder().find(f)[0] + >>> doctest.DocTestRunner(verbose=False).run(test) + TestResults(failed=0, attempted=2) + +Note that a trailing colon doesn't matter either: + + >>> def f(x): + ... r''' + ... >>> from http.client import HTTPException + ... >>> raise HTTPException() #doctest: +IGNORE_EXCEPTION_DETAIL + ... Traceback (most recent call last): + ... foo.bar.HTTPException: + ... ''' + >>> test = doctest.DocTestFinder().find(f)[0] + >>> doctest.DocTestRunner(verbose=False).run(test) + TestResults(failed=0, attempted=2) + +If an exception is raised but not expected, then it is reported as an +unexpected exception: + + >>> def f(x): + ... r''' + ... >>> 1//0 + ... 0 + ... ''' + >>> test = doctest.DocTestFinder().find(f)[0] + >>> doctest.DocTestRunner(verbose=False).run(test) + ... # doctest: +ELLIPSIS + ********************************************************************** + File ..., line 3, in f + Failed example: + 1//0 + Exception raised: + Traceback (most recent call last): + ... + ZeroDivisionError: integer division or modulo by zero + TestResults(failed=1, attempted=1) + + >>> _colorize.COLORIZE = save_colorize +""" + def displayhook(): r""" +Test that changing sys.displayhook doesn't matter for doctest. + + >>> import sys + >>> orig_displayhook = sys.displayhook + >>> def my_displayhook(x): + ... print('hi!') + >>> sys.displayhook = my_displayhook + >>> def f(): + ... ''' + ... >>> 3 + ... 3 + ... ''' + >>> test = doctest.DocTestFinder().find(f)[0] + >>> r = doctest.DocTestRunner(verbose=False).run(test) + >>> post_displayhook = sys.displayhook + + We need to restore sys.displayhook now, so that we'll be able to test + results. + + >>> sys.displayhook = orig_displayhook + + Ok, now we can check that everything is ok. + + >>> r + TestResults(failed=0, attempted=1) + >>> post_displayhook is my_displayhook + True +""" + def optionflags(): r""" +Tests of `DocTestRunner`'s option flag handling. + +Several option flags can be used to customize the behavior of the test +runner. These are defined as module constants in doctest, and passed +to the DocTestRunner constructor (multiple constants should be ORed +together). + +The DONT_ACCEPT_TRUE_FOR_1 flag disables matches between True/False +and 1/0: + + >>> save_colorize = _colorize.COLORIZE + >>> _colorize.COLORIZE = False + + >>> def f(x): + ... '>>> True\n1\n' + + >>> # Without the flag: + >>> test = doctest.DocTestFinder().find(f)[0] + >>> doctest.DocTestRunner(verbose=False).run(test) + TestResults(failed=0, attempted=1) + + >>> # With the flag: + >>> test = doctest.DocTestFinder().find(f)[0] + >>> flags = doctest.DONT_ACCEPT_TRUE_FOR_1 + >>> doctest.DocTestRunner(verbose=False, optionflags=flags).run(test) + ... # doctest: +ELLIPSIS + ********************************************************************** + File ..., line 2, in f + Failed example: + True + Expected: + 1 + Got: + True + TestResults(failed=1, attempted=1) + +The DONT_ACCEPT_BLANKLINE flag disables the match between blank lines +and the '' marker: + + >>> def f(x): + ... '>>> print("a\\n\\nb")\na\n\nb\n' + + >>> # Without the flag: + >>> test = doctest.DocTestFinder().find(f)[0] + >>> doctest.DocTestRunner(verbose=False).run(test) + TestResults(failed=0, attempted=1) + + >>> # With the flag: + >>> test = doctest.DocTestFinder().find(f)[0] + >>> flags = doctest.DONT_ACCEPT_BLANKLINE + >>> doctest.DocTestRunner(verbose=False, optionflags=flags).run(test) + ... # doctest: +ELLIPSIS + ********************************************************************** + File ..., line 2, in f + Failed example: + print("a\n\nb") + Expected: + a + + b + Got: + a + + b + TestResults(failed=1, attempted=1) + +The NORMALIZE_WHITESPACE flag causes all sequences of whitespace to be +treated as equal: + + >>> def f(x): + ... '\n>>> print(1, 2, 3)\n 1 2\n 3' + + >>> # Without the flag: + >>> test = doctest.DocTestFinder().find(f)[0] + >>> doctest.DocTestRunner(verbose=False).run(test) + ... # doctest: +ELLIPSIS + ********************************************************************** + File ..., line 3, in f + Failed example: + print(1, 2, 3) + Expected: + 1 2 + 3 + Got: + 1 2 3 + TestResults(failed=1, attempted=1) + + >>> # With the flag: + >>> test = doctest.DocTestFinder().find(f)[0] + >>> flags = doctest.NORMALIZE_WHITESPACE + >>> doctest.DocTestRunner(verbose=False, optionflags=flags).run(test) + TestResults(failed=0, attempted=1) + + An example from the docs: + >>> print(list(range(20))) #doctest: +NORMALIZE_WHITESPACE + [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, + 10, 11, 12, 13, 14, 15, 16, 17, 18, 19] + +The ELLIPSIS flag causes ellipsis marker ("...") in the expected +output to match any substring in the actual output: + + >>> def f(x): + ... '>>> print(list(range(15)))\n[0, 1, 2, ..., 14]\n' + + >>> # Without the flag: + >>> test = doctest.DocTestFinder().find(f)[0] + >>> doctest.DocTestRunner(verbose=False).run(test) + ... # doctest: +ELLIPSIS + ********************************************************************** + File ..., line 2, in f + Failed example: + print(list(range(15))) + Expected: + [0, 1, 2, ..., 14] + Got: + [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14] + TestResults(failed=1, attempted=1) + + >>> # With the flag: + >>> test = doctest.DocTestFinder().find(f)[0] + >>> flags = doctest.ELLIPSIS + >>> doctest.DocTestRunner(verbose=False, optionflags=flags).run(test) + TestResults(failed=0, attempted=1) + + ... also matches nothing: + + >>> if 1: + ... for i in range(100): + ... print(i**2, end=' ') #doctest: +ELLIPSIS + ... print('!') + 0 1...4...9 16 ... 36 49 64 ... 9801 ! + + ... can be surprising; e.g., this test passes: + + >>> if 1: #doctest: +ELLIPSIS + ... for i in range(20): + ... print(i, end=' ') + ... print(20) + 0 1 2 ...1...2...0 + + Examples from the docs: + + >>> print(list(range(20))) # doctest:+ELLIPSIS + [0, 1, ..., 18, 19] + + >>> print(list(range(20))) # doctest: +ELLIPSIS + ... # doctest: +NORMALIZE_WHITESPACE + [0, 1, ..., 18, 19] + +The SKIP flag causes an example to be skipped entirely. I.e., the +example is not run. It can be useful in contexts where doctest +examples serve as both documentation and test cases, and an example +should be included for documentation purposes, but should not be +checked (e.g., because its output is random, or depends on resources +which would be unavailable.) The SKIP flag can also be used for +'commenting out' broken examples. + + >>> import unavailable_resource # doctest: +SKIP + >>> unavailable_resource.do_something() # doctest: +SKIP + >>> unavailable_resource.blow_up() # doctest: +SKIP + Traceback (most recent call last): + ... + UncheckedBlowUpError: Nobody checks me. + + >>> import random + >>> print(random.random()) # doctest: +SKIP + 0.721216923889 + +The REPORT_UDIFF flag causes failures that involve multi-line expected +and actual outputs to be displayed using a unified diff: + + >>> def f(x): + ... r''' + ... >>> print('\n'.join('abcdefg')) + ... a + ... B + ... c + ... d + ... f + ... g + ... h + ... ''' + + >>> # Without the flag: + >>> test = doctest.DocTestFinder().find(f)[0] + >>> doctest.DocTestRunner(verbose=False).run(test) + ... # doctest: +ELLIPSIS + ********************************************************************** + File ..., line 3, in f + Failed example: + print('\n'.join('abcdefg')) + Expected: + a + B + c + d + f + g + h + Got: + a + b + c + d + e + f + g + TestResults(failed=1, attempted=1) + + >>> # With the flag: + >>> test = doctest.DocTestFinder().find(f)[0] + >>> flags = doctest.REPORT_UDIFF + >>> doctest.DocTestRunner(verbose=False, optionflags=flags).run(test) + ... # doctest: +ELLIPSIS + ********************************************************************** + File ..., line 3, in f + Failed example: + print('\n'.join('abcdefg')) + Differences (unified diff with -expected +actual): + @@ -1,7 +1,7 @@ + a + -B + +b + c + d + +e + f + g + -h + TestResults(failed=1, attempted=1) + +The REPORT_CDIFF flag causes failures that involve multi-line expected +and actual outputs to be displayed using a context diff: + + >>> # Reuse f() from the REPORT_UDIFF example, above. + >>> test = doctest.DocTestFinder().find(f)[0] + >>> flags = doctest.REPORT_CDIFF + >>> doctest.DocTestRunner(verbose=False, optionflags=flags).run(test) + ... # doctest: +ELLIPSIS + ********************************************************************** + File ..., line 3, in f + Failed example: + print('\n'.join('abcdefg')) + Differences (context diff with expected followed by actual): + *************** + *** 1,7 **** + a + ! B + c + d + f + g + - h + --- 1,7 ---- + a + ! b + c + d + + e + f + g + TestResults(failed=1, attempted=1) + + +The REPORT_NDIFF flag causes failures to use the difflib.Differ algorithm +used by the popular ndiff.py utility. This does intraline difference +marking, as well as interline differences. + + >>> def f(x): + ... r''' + ... >>> print("a b c d e f g h i j k l m") + ... a b c d e f g h i j k 1 m + ... ''' + >>> test = doctest.DocTestFinder().find(f)[0] + >>> flags = doctest.REPORT_NDIFF + >>> doctest.DocTestRunner(verbose=False, optionflags=flags).run(test) + ... # doctest: +ELLIPSIS + ********************************************************************** + File ..., line 3, in f + Failed example: + print("a b c d e f g h i j k l m") + Differences (ndiff with -expected +actual): + - a b c d e f g h i j k 1 m + ? ^ + + a b c d e f g h i j k l m + ? + ++ ^ + TestResults(failed=1, attempted=1) + +The REPORT_ONLY_FIRST_FAILURE suppresses result output after the first +failing example: + + >>> def f(x): + ... r''' + ... >>> print(1) # first success + ... 1 + ... >>> print(2) # first failure + ... 200 + ... >>> print(3) # second failure + ... 300 + ... >>> print(4) # second success + ... 4 + ... >>> print(5) # third failure + ... 500 + ... ''' + >>> test = doctest.DocTestFinder().find(f)[0] + >>> flags = doctest.REPORT_ONLY_FIRST_FAILURE + >>> doctest.DocTestRunner(verbose=False, optionflags=flags).run(test) + ... # doctest: +ELLIPSIS + ********************************************************************** + File ..., line 5, in f + Failed example: + print(2) # first failure + Expected: + 200 + Got: + 2 + TestResults(failed=3, attempted=5) + +However, output from `report_start` is not suppressed: + + >>> doctest.DocTestRunner(verbose=True, optionflags=flags).run(test) + ... # doctest: +ELLIPSIS + Trying: + print(1) # first success + Expecting: + 1 + ok + Trying: + print(2) # first failure + Expecting: + 200 + ********************************************************************** + File ..., line 5, in f + Failed example: + print(2) # first failure + Expected: + 200 + Got: + 2 + TestResults(failed=3, attempted=5) + +The FAIL_FAST flag causes the runner to exit after the first failing example, +so subsequent examples are not even attempted: + + >>> flags = doctest.FAIL_FAST + >>> doctest.DocTestRunner(verbose=False, optionflags=flags).run(test) + ... # doctest: +ELLIPSIS + ********************************************************************** + File ..., line 5, in f + Failed example: + print(2) # first failure + Expected: + 200 + Got: + 2 + TestResults(failed=1, attempted=2) + +Specifying both FAIL_FAST and REPORT_ONLY_FIRST_FAILURE is equivalent to +FAIL_FAST only: + + >>> flags = doctest.FAIL_FAST | doctest.REPORT_ONLY_FIRST_FAILURE + >>> doctest.DocTestRunner(verbose=False, optionflags=flags).run(test) + ... # doctest: +ELLIPSIS + ********************************************************************** + File ..., line 5, in f + Failed example: + print(2) # first failure + Expected: + 200 + Got: + 2 + TestResults(failed=1, attempted=2) + +For the purposes of both REPORT_ONLY_FIRST_FAILURE and FAIL_FAST, unexpected +exceptions count as failures: + + >>> def f(x): + ... r''' + ... >>> print(1) # first success + ... 1 + ... >>> raise ValueError(2) # first failure + ... 200 + ... >>> print(3) # second failure + ... 300 + ... >>> print(4) # second success + ... 4 + ... >>> print(5) # third failure + ... 500 + ... ''' + >>> test = doctest.DocTestFinder().find(f)[0] + >>> flags = doctest.REPORT_ONLY_FIRST_FAILURE + >>> doctest.DocTestRunner(verbose=False, optionflags=flags).run(test) + ... # doctest: +ELLIPSIS + ********************************************************************** + File ..., line 5, in f + Failed example: + raise ValueError(2) # first failure + Exception raised: + ... + ValueError: 2 + TestResults(failed=3, attempted=5) + >>> flags = doctest.FAIL_FAST + >>> doctest.DocTestRunner(verbose=False, optionflags=flags).run(test) + ... # doctest: +ELLIPSIS + ********************************************************************** + File ..., line 5, in f + Failed example: + raise ValueError(2) # first failure + Exception raised: + ... + ValueError: 2 + TestResults(failed=1, attempted=2) + +New option flags can also be registered, via register_optionflag(). Here +we reach into doctest's internals a bit. + + >>> unlikely = "UNLIKELY_OPTION_NAME" + >>> unlikely in doctest.OPTIONFLAGS_BY_NAME + False + >>> new_flag_value = doctest.register_optionflag(unlikely) + >>> unlikely in doctest.OPTIONFLAGS_BY_NAME + True + +Before 2.4.4/2.5, registering a name more than once erroneously created +more than one flag value. Here we verify that's fixed: + + >>> redundant_flag_value = doctest.register_optionflag(unlikely) + >>> redundant_flag_value == new_flag_value + True + +Clean up. + >>> del doctest.OPTIONFLAGS_BY_NAME[unlikely] + >>> _colorize.COLORIZE = save_colorize + + """ + + def option_directives(): r""" +Tests of `DocTestRunner`'s option directive mechanism. + +Option directives can be used to turn option flags on or off for a +single example. To turn an option on for an example, follow that +example with a comment of the form ``# doctest: +OPTION``: + + >>> save_colorize = _colorize.COLORIZE + >>> _colorize.COLORIZE = False + + >>> def f(x): r''' + ... >>> print(list(range(10))) # should fail: no ellipsis + ... [0, 1, ..., 9] + ... + ... >>> print(list(range(10))) # doctest: +ELLIPSIS + ... [0, 1, ..., 9] + ... ''' + >>> test = doctest.DocTestFinder().find(f)[0] + >>> doctest.DocTestRunner(verbose=False).run(test) + ... # doctest: +ELLIPSIS + ********************************************************************** + File ..., line 2, in f + Failed example: + print(list(range(10))) # should fail: no ellipsis + Expected: + [0, 1, ..., 9] + Got: + [0, 1, 2, 3, 4, 5, 6, 7, 8, 9] + TestResults(failed=1, attempted=2) + +To turn an option off for an example, follow that example with a +comment of the form ``# doctest: -OPTION``: + + >>> def f(x): r''' + ... >>> print(list(range(10))) + ... [0, 1, ..., 9] + ... + ... >>> # should fail: no ellipsis + ... >>> print(list(range(10))) # doctest: -ELLIPSIS + ... [0, 1, ..., 9] + ... ''' + >>> test = doctest.DocTestFinder().find(f)[0] + >>> doctest.DocTestRunner(verbose=False, + ... optionflags=doctest.ELLIPSIS).run(test) + ... # doctest: +ELLIPSIS + ********************************************************************** + File ..., line 6, in f + Failed example: + print(list(range(10))) # doctest: -ELLIPSIS + Expected: + [0, 1, ..., 9] + Got: + [0, 1, 2, 3, 4, 5, 6, 7, 8, 9] + TestResults(failed=1, attempted=2) + +Option directives affect only the example that they appear with; they +do not change the options for surrounding examples: + + >>> def f(x): r''' + ... >>> print(list(range(10))) # Should fail: no ellipsis + ... [0, 1, ..., 9] + ... + ... >>> print(list(range(10))) # doctest: +ELLIPSIS + ... [0, 1, ..., 9] + ... + ... >>> print(list(range(10))) # Should fail: no ellipsis + ... [0, 1, ..., 9] + ... ''' + >>> test = doctest.DocTestFinder().find(f)[0] + >>> doctest.DocTestRunner(verbose=False).run(test) + ... # doctest: +ELLIPSIS + ********************************************************************** + File ..., line 2, in f + Failed example: + print(list(range(10))) # Should fail: no ellipsis + Expected: + [0, 1, ..., 9] + Got: + [0, 1, 2, 3, 4, 5, 6, 7, 8, 9] + ********************************************************************** + File ..., line 8, in f + Failed example: + print(list(range(10))) # Should fail: no ellipsis + Expected: + [0, 1, ..., 9] + Got: + [0, 1, 2, 3, 4, 5, 6, 7, 8, 9] + TestResults(failed=2, attempted=3) + +Multiple options may be modified by a single option directive. They +may be separated by whitespace, commas, or both: + + >>> def f(x): r''' + ... >>> print(list(range(10))) # Should fail + ... [0, 1, ..., 9] + ... >>> print(list(range(10))) # Should succeed + ... ... # doctest: +ELLIPSIS +NORMALIZE_WHITESPACE + ... [0, 1, ..., 9] + ... ''' + >>> test = doctest.DocTestFinder().find(f)[0] + >>> doctest.DocTestRunner(verbose=False).run(test) + ... # doctest: +ELLIPSIS + ********************************************************************** + File ..., line 2, in f + Failed example: + print(list(range(10))) # Should fail + Expected: + [0, 1, ..., 9] + Got: + [0, 1, 2, 3, 4, 5, 6, 7, 8, 9] + TestResults(failed=1, attempted=2) + + >>> def f(x): r''' + ... >>> print(list(range(10))) # Should fail + ... [0, 1, ..., 9] + ... >>> print(list(range(10))) # Should succeed + ... ... # doctest: +ELLIPSIS,+NORMALIZE_WHITESPACE + ... [0, 1, ..., 9] + ... ''' + >>> test = doctest.DocTestFinder().find(f)[0] + >>> doctest.DocTestRunner(verbose=False).run(test) + ... # doctest: +ELLIPSIS + ********************************************************************** + File ..., line 2, in f + Failed example: + print(list(range(10))) # Should fail + Expected: + [0, 1, ..., 9] + Got: + [0, 1, 2, 3, 4, 5, 6, 7, 8, 9] + TestResults(failed=1, attempted=2) + + >>> def f(x): r''' + ... >>> print(list(range(10))) # Should fail + ... [0, 1, ..., 9] + ... >>> print(list(range(10))) # Should succeed + ... ... # doctest: +ELLIPSIS, +NORMALIZE_WHITESPACE + ... [0, 1, ..., 9] + ... ''' + >>> test = doctest.DocTestFinder().find(f)[0] + >>> doctest.DocTestRunner(verbose=False).run(test) + ... # doctest: +ELLIPSIS + ********************************************************************** + File ..., line 2, in f + Failed example: + print(list(range(10))) # Should fail + Expected: + [0, 1, ..., 9] + Got: + [0, 1, 2, 3, 4, 5, 6, 7, 8, 9] + TestResults(failed=1, attempted=2) + +The option directive may be put on the line following the source, as +long as a continuation prompt is used: + + >>> def f(x): r''' + ... >>> print(list(range(10))) + ... ... # doctest: +ELLIPSIS + ... [0, 1, ..., 9] + ... ''' + >>> test = doctest.DocTestFinder().find(f)[0] + >>> doctest.DocTestRunner(verbose=False).run(test) + TestResults(failed=0, attempted=1) + +For examples with multi-line source, the option directive may appear +at the end of any line: + + >>> def f(x): r''' + ... >>> for x in range(10): # doctest: +ELLIPSIS + ... ... print(' ', x, end='', sep='') + ... 0 1 2 ... 9 + ... + ... >>> for x in range(10): + ... ... print(' ', x, end='', sep='') # doctest: +ELLIPSIS + ... 0 1 2 ... 9 + ... ''' + >>> test = doctest.DocTestFinder().find(f)[0] + >>> doctest.DocTestRunner(verbose=False).run(test) + TestResults(failed=0, attempted=2) + +If more than one line of an example with multi-line source has an +option directive, then they are combined: + + >>> def f(x): r''' + ... Should fail (option directive not on the last line): + ... >>> for x in range(10): # doctest: +ELLIPSIS + ... ... print(x, end=' ') # doctest: +NORMALIZE_WHITESPACE + ... 0 1 2...9 + ... ''' + >>> test = doctest.DocTestFinder().find(f)[0] + >>> doctest.DocTestRunner(verbose=False).run(test) + TestResults(failed=0, attempted=1) + +It is an error to have a comment of the form ``# doctest:`` that is +*not* followed by words of the form ``+OPTION`` or ``-OPTION``, where +``OPTION`` is an option that has been registered with +`register_option`: + + >>> # Error: Option not registered + >>> s = '>>> print(12) #doctest: +BADOPTION' + >>> test = doctest.DocTestParser().get_doctest(s, {}, 's', 's.py', 0) + Traceback (most recent call last): + ValueError: line 1 of the doctest for s has an invalid option: '+BADOPTION' + + >>> # Error: No + or - prefix + >>> s = '>>> print(12) #doctest: ELLIPSIS' + >>> test = doctest.DocTestParser().get_doctest(s, {}, 's', 's.py', 0) + Traceback (most recent call last): + ValueError: line 1 of the doctest for s has an invalid option: 'ELLIPSIS' + +It is an error to use an option directive on a line that contains no +source: + + >>> s = '>>> # doctest: +ELLIPSIS' + >>> test = doctest.DocTestParser().get_doctest(s, {}, 's', 's.py', 0) + Traceback (most recent call last): + ValueError: line 0 of the doctest for s has an option directive on a line with no example: '# doctest: +ELLIPSIS' + + >>> _colorize.COLORIZE = save_colorize +""" + +def test_testsource(): r""" +Unit tests for `testsource()`. + +The testsource() function takes a module and a name, finds the (first) +test with that name in that module, and converts it to a script. The +example code is converted to regular Python code. The surrounding +words and expected output are converted to comments: + + >>> from test.test_doctest import test_doctest + >>> name = 'test.test_doctest.test_doctest.sample_func' + >>> print(doctest.testsource(test_doctest, name)) + # Blah blah + # + print(sample_func(22)) + # Expected: + ## 44 + # + # Yee ha! + + + >>> name = 'test.test_doctest.test_doctest.SampleNewStyleClass' + >>> print(doctest.testsource(test_doctest, name)) + print('1\n2\n3') + # Expected: + ## 1 + ## 2 + ## 3 + + + >>> name = 'test.test_doctest.test_doctest.SampleClass.a_classmethod' + >>> print(doctest.testsource(test_doctest, name)) + print(SampleClass.a_classmethod(10)) + # Expected: + ## 12 + print(SampleClass(0).a_classmethod(10)) + # Expected: + ## 12 + +""" + +# TODO: RUSTPYTHON +# Issue with pdb +# def test_debug(): r""" + +# Create a docstring that we want to debug: + +# >>> s = ''' +# ... >>> x = 12 +# ... >>> print(x) +# ... 12 +# ... ''' + +# Create some fake stdin input, to feed to the debugger: + +# >>> real_stdin = sys.stdin +# >>> sys.stdin = FakeInput(['next', 'print(x)', 'continue']) + +# Run the debugger on the docstring, and then restore sys.stdin. + +# >>> try: doctest.debug_src(s) +# ... finally: sys.stdin = real_stdin +# > (1)() +# (Pdb) next +# 12 +# --Return-- +# > (1)()->None +# (Pdb) print(x) +# 12 +# (Pdb) continue + +# """ + +# TODO: RUSTPYTHON +# Issue with pdb +# AttributeError: 'frame' object has no attribute 'f_trace_opcodes'. Did you mean: 'f_trace_lines'? +# if not hasattr(sys, 'gettrace') or not sys.gettrace(): +# def test_pdb_set_trace(): +# """Using pdb.set_trace from a doctest. + +# You can use pdb.set_trace from a doctest. To do so, you must +# retrieve the set_trace function from the pdb module at the time +# you use it. The doctest module changes sys.stdout so that it can +# capture program output. It also temporarily replaces pdb.set_trace +# with a version that restores stdout. This is necessary for you to +# see debugger output. + +# >>> save_colorize = _colorize.COLORIZE +# >>> _colorize.COLORIZE = False + +# >>> doc = ''' +# ... >>> x = 42 +# ... >>> raise Exception('clé') +# ... Traceback (most recent call last): +# ... Exception: clé +# ... >>> import pdb; pdb.set_trace() +# ... ''' +# >>> parser = doctest.DocTestParser() +# >>> test = parser.get_doctest(doc, {}, "foo-bar@baz", "foo-bar@baz.py", 0) +# >>> runner = doctest.DocTestRunner(verbose=False) + +# To demonstrate this, we'll create a fake standard input that +# captures our debugger input: + +# >>> real_stdin = sys.stdin +# >>> sys.stdin = FakeInput([ +# ... 'print(x)', # print data defined by the example +# ... 'continue', # stop debugging +# ... '']) + +# >>> try: runner.run(test) +# ... finally: sys.stdin = real_stdin +# > (1)() +# -> import pdb; pdb.set_trace() +# (Pdb) print(x) +# 42 +# (Pdb) continue +# TestResults(failed=0, attempted=3) + +# You can also put pdb.set_trace in a function called from a test: + +# >>> def calls_set_trace(): +# ... y=2 +# ... import pdb; pdb.set_trace() + +# >>> doc = ''' +# ... >>> x=1 +# ... >>> calls_set_trace() +# ... ''' +# >>> test = parser.get_doctest(doc, globals(), "foo-bar@baz", "foo-bar@baz.py", 0) +# >>> real_stdin = sys.stdin +# >>> sys.stdin = FakeInput([ +# ... 'print(y)', # print data defined in the function +# ... 'up', # out of function +# ... 'print(x)', # print data defined by the example +# ... 'continue', # stop debugging +# ... '']) + +# >>> try: +# ... runner.run(test) +# ... finally: +# ... sys.stdin = real_stdin +# > (3)calls_set_trace() +# -> import pdb; pdb.set_trace() +# (Pdb) print(y) +# 2 +# (Pdb) up +# > (1)() +# -> calls_set_trace() +# (Pdb) print(x) +# 1 +# (Pdb) continue +# TestResults(failed=0, attempted=2) + +# During interactive debugging, source code is shown, even for +# doctest examples: + +# >>> doc = ''' +# ... >>> def f(x): +# ... ... g(x*2) +# ... >>> def g(x): +# ... ... print(x+3) +# ... ... import pdb; pdb.set_trace() +# ... >>> f(3) +# ... ''' +# >>> test = parser.get_doctest(doc, globals(), "foo-bar@baz", "foo-bar@baz.py", 0) +# >>> real_stdin = sys.stdin +# >>> sys.stdin = FakeInput([ +# ... 'step', # return event of g +# ... 'list', # list source from example 2 +# ... 'next', # return from g() +# ... 'list', # list source from example 1 +# ... 'next', # return from f() +# ... 'list', # list source from example 3 +# ... 'continue', # stop debugging +# ... '']) +# >>> try: runner.run(test) +# ... finally: sys.stdin = real_stdin +# ... # doctest: +NORMALIZE_WHITESPACE +# > (3)g() +# -> import pdb; pdb.set_trace() +# (Pdb) step +# --Return-- +# > (3)g()->None +# -> import pdb; pdb.set_trace() +# (Pdb) list +# 1 def g(x): +# 2 print(x+3) +# 3 -> import pdb; pdb.set_trace() +# [EOF] +# (Pdb) next +# --Return-- +# > (2)f()->None +# -> g(x*2) +# (Pdb) list +# 1 def f(x): +# 2 -> g(x*2) +# [EOF] +# (Pdb) next +# --Return-- +# > (1)()->None +# -> f(3) +# (Pdb) list +# 1 -> f(3) +# [EOF] +# (Pdb) continue +# ********************************************************************** +# File "foo-bar@baz.py", line 7, in foo-bar@baz +# Failed example: +# f(3) +# Expected nothing +# Got: +# 9 +# TestResults(failed=1, attempted=3) + +# >>> _colorize.COLORIZE = save_colorize +# """ + + # TODO: RUSTPYTHON + # Issue with pdb + # AttributeError: 'frame' object has no attribute 'f_trace_opcodes'. Did you mean: 'f_trace_lines'? + # def test_pdb_set_trace_nested(): + # """This illustrates more-demanding use of set_trace with nested functions. + + # >>> class C(object): + # ... def calls_set_trace(self): + # ... y = 1 + # ... import pdb; pdb.set_trace() + # ... self.f1() + # ... y = 2 + # ... def f1(self): + # ... x = 1 + # ... self.f2() + # ... x = 2 + # ... def f2(self): + # ... z = 1 + # ... z = 2 + + # >>> calls_set_trace = C().calls_set_trace + + # >>> doc = ''' + # ... >>> a = 1 + # ... >>> calls_set_trace() + # ... ''' + # >>> parser = doctest.DocTestParser() + # >>> runner = doctest.DocTestRunner(verbose=False) + # >>> test = parser.get_doctest(doc, globals(), "foo-bar@baz", "foo-bar@baz.py", 0) + # >>> real_stdin = sys.stdin + # >>> sys.stdin = FakeInput([ + # ... 'step', + # ... 'print(y)', # print data defined in the function + # ... 'step', 'step', 'step', 'step', 'step', 'step', 'print(z)', + # ... 'up', 'print(x)', + # ... 'up', 'print(y)', + # ... 'up', 'print(foo)', + # ... 'continue', # stop debugging + # ... '']) + + # >>> try: + # ... runner.run(test) + # ... finally: + # ... sys.stdin = real_stdin + # ... # doctest: +REPORT_NDIFF + # > (4)calls_set_trace() + # -> import pdb; pdb.set_trace() + # (Pdb) step + # > (5)calls_set_trace() + # -> self.f1() + # (Pdb) print(y) + # 1 + # (Pdb) step + # --Call-- + # > (7)f1() + # -> def f1(self): + # (Pdb) step + # > (8)f1() + # -> x = 1 + # (Pdb) step + # > (9)f1() + # -> self.f2() + # (Pdb) step + # --Call-- + # > (11)f2() + # -> def f2(self): + # (Pdb) step + # > (12)f2() + # -> z = 1 + # (Pdb) step + # > (13)f2() + # -> z = 2 + # (Pdb) print(z) + # 1 + # (Pdb) up + # > (9)f1() + # -> self.f2() + # (Pdb) print(x) + # 1 + # (Pdb) up + # > (5)calls_set_trace() + # -> self.f1() + # (Pdb) print(y) + # 1 + # (Pdb) up + # > (1)() + # -> calls_set_trace() + # (Pdb) print(foo) + # *** NameError: name 'foo' is not defined + # (Pdb) continue + # TestResults(failed=0, attempted=2) + # """ + +# TODO: RUSTPYTHON +# Issue with pdb +# AttributeError: 'frame' object has no attribute 'f_trace_opcodes'. Did you mean: 'f_trace_lines'? +# def test_DocTestSuite(): +# """DocTestSuite creates a unittest test suite from a doctest. + +# We create a Suite by providing a module. A module can be provided +# by passing a module object: + +# >>> import unittest +# >>> import test.test_doctest.sample_doctest +# >>> suite = doctest.DocTestSuite(test.test_doctest.sample_doctest) +# >>> result = suite.run(unittest.TestResult()) +# >>> result +# +# >>> for tst, _ in result.failures: +# ... print(tst) +# bad (test.test_doctest.sample_doctest.__test__) +# foo (test.test_doctest.sample_doctest) +# test_silly_setup (test.test_doctest.sample_doctest) +# y_is_one (test.test_doctest.sample_doctest) + +# We can also supply the module by name: + +# >>> suite = doctest.DocTestSuite('test.test_doctest.sample_doctest') +# >>> result = suite.run(unittest.TestResult()) +# >>> result +# + +# The module need not contain any doctest examples: + +# >>> suite = doctest.DocTestSuite('test.test_doctest.sample_doctest_no_doctests') +# >>> suite.run(unittest.TestResult()) +# + +# The module need not contain any docstrings either: + +# >>> suite = doctest.DocTestSuite('test.test_doctest.sample_doctest_no_docstrings') +# >>> suite.run(unittest.TestResult()) +# + +# If all examples in a docstring are skipped, unittest will report it as a +# skipped test: + +# >>> suite = doctest.DocTestSuite('test.test_doctest.sample_doctest_skip') +# >>> result = suite.run(unittest.TestResult()) +# >>> result +# +# >>> len(result.skipped) +# 2 +# >>> for tst, _ in result.skipped: +# ... print(tst) +# double_skip (test.test_doctest.sample_doctest_skip) +# single_skip (test.test_doctest.sample_doctest_skip) +# >>> for tst, _ in result.failures: +# ... print(tst) +# no_skip_fail (test.test_doctest.sample_doctest_skip) +# partial_skip_fail (test.test_doctest.sample_doctest_skip) + +# We can use the current module: + +# >>> suite = test.test_doctest.sample_doctest.test_suite() +# >>> suite.run(unittest.TestResult()) +# + +# We can also provide a DocTestFinder: + +# >>> finder = doctest.DocTestFinder() +# >>> suite = doctest.DocTestSuite('test.test_doctest.sample_doctest', +# ... test_finder=finder) +# >>> suite.run(unittest.TestResult()) +# + +# The DocTestFinder need not return any tests: + +# >>> finder = doctest.DocTestFinder() +# >>> suite = doctest.DocTestSuite('test.test_doctest.sample_doctest_no_docstrings', +# ... test_finder=finder) +# >>> suite.run(unittest.TestResult()) +# + +# We can supply global variables. If we pass globs, they will be +# used instead of the module globals. Here we'll pass an empty +# globals, triggering an extra error: + +# >>> suite = doctest.DocTestSuite('test.test_doctest.sample_doctest', globs={}) +# >>> suite.run(unittest.TestResult()) +# + +# Alternatively, we can provide extra globals. Here we'll make an +# error go away by providing an extra global variable: + +# >>> suite = doctest.DocTestSuite('test.test_doctest.sample_doctest', +# ... extraglobs={'y': 1}) +# >>> suite.run(unittest.TestResult()) +# + +# You can pass option flags. Here we'll cause an extra error +# by disabling the blank-line feature: + +# >>> suite = doctest.DocTestSuite('test.test_doctest.sample_doctest', +# ... optionflags=doctest.DONT_ACCEPT_BLANKLINE) +# >>> suite.run(unittest.TestResult()) +# + +# You can supply setUp and tearDown functions: + +# >>> def setUp(t): +# ... from test.test_doctest import test_doctest +# ... test_doctest.sillySetup = True + +# >>> def tearDown(t): +# ... from test.test_doctest import test_doctest +# ... del test_doctest.sillySetup + +# Here, we installed a silly variable that the test expects: + +# >>> suite = doctest.DocTestSuite('test.test_doctest.sample_doctest', +# ... setUp=setUp, tearDown=tearDown) +# >>> suite.run(unittest.TestResult()) +# + +# But the tearDown restores sanity: + +# >>> from test.test_doctest import test_doctest +# >>> test_doctest.sillySetup +# Traceback (most recent call last): +# ... +# AttributeError: module 'test.test_doctest.test_doctest' has no attribute 'sillySetup' + +# The setUp and tearDown functions are passed test objects. Here +# we'll use the setUp function to supply the missing variable y: + +# >>> def setUp(test): +# ... test.globs['y'] = 1 + +# >>> suite = doctest.DocTestSuite('test.test_doctest.sample_doctest', setUp=setUp) +# >>> suite.run(unittest.TestResult()) +# + +# Here, we didn't need to use a tearDown function because we +# modified the test globals, which are a copy of the +# sample_doctest module dictionary. The test globals are +# automatically cleared for us after a test. +# """ + +# TODO: RUSTPYTHON +# traceback + error message is different than in CPython +# def test_DocTestSuite_errors(): +# """Tests for error reporting in DocTestSuite. + +# >>> import unittest +# >>> import test.test_doctest.sample_doctest_errors as mod +# >>> suite = doctest.DocTestSuite(mod) +# >>> result = suite.run(unittest.TestResult()) +# >>> result +# +# >>> print(result.failures[0][1]) # doctest: +ELLIPSIS +# Traceback (most recent call last): +# File ... +# raise self.failureException(self.format_failure(new.getvalue())) +# AssertionError: Failed doctest test for test.test_doctest.sample_doctest_errors +# File "...sample_doctest_errors.py", line 0, in sample_doctest_errors +# +# ---------------------------------------------------------------------- +# File "...sample_doctest_errors.py", line 5, in test.test_doctest.sample_doctest_errors +# Failed example: +# 2 + 2 +# Expected: +# 5 +# Got: +# 4 +# ---------------------------------------------------------------------- +# File "...sample_doctest_errors.py", line 7, in test.test_doctest.sample_doctest_errors +# Failed example: +# 1/0 +# Exception raised: +# Traceback (most recent call last): +# File ... +# exec(compile(example.source, filename, "single", +# ~~~~^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +# compileflags, True), test.globs) +# ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +# File "", line 1, in +# 1/0 +# ~^~ +# ZeroDivisionError: division by zero +# +# +# >>> print(result.failures[1][1]) # doctest: +ELLIPSIS +# Traceback (most recent call last): +# File ... +# raise self.failureException(self.format_failure(new.getvalue())) +# AssertionError: Failed doctest test for test.test_doctest.sample_doctest_errors.__test__.bad +# File "...sample_doctest_errors.py", line unknown line number, in bad +# +# ---------------------------------------------------------------------- +# File "...sample_doctest_errors.py", line ?, in test.test_doctest.sample_doctest_errors.__test__.bad +# Failed example: +# 2 + 2 +# Expected: +# 5 +# Got: +# 4 +# ---------------------------------------------------------------------- +# File "...sample_doctest_errors.py", line ?, in test.test_doctest.sample_doctest_errors.__test__.bad +# Failed example: +# 1/0 +# Exception raised: +# Traceback (most recent call last): +# File ... +# exec(compile(example.source, filename, "single", +# ~~~~^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +# compileflags, True), test.globs) +# ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +# File "", line 1, in +# 1/0 +# ~^~ +# ZeroDivisionError: division by zero +# +# +# >>> print(result.failures[2][1]) # doctest: +ELLIPSIS +# Traceback (most recent call last): +# File ... +# raise self.failureException(self.format_failure(new.getvalue())) +# AssertionError: Failed doctest test for test.test_doctest.sample_doctest_errors.errors +# File "...sample_doctest_errors.py", line 14, in errors +# +# ---------------------------------------------------------------------- +# File "...sample_doctest_errors.py", line 16, in test.test_doctest.sample_doctest_errors.errors +# Failed example: +# 2 + 2 +# Expected: +# 5 +# Got: +# 4 +# ---------------------------------------------------------------------- +# File "...sample_doctest_errors.py", line 18, in test.test_doctest.sample_doctest_errors.errors +# Failed example: +# 1/0 +# Exception raised: +# Traceback (most recent call last): +# File ... +# exec(compile(example.source, filename, "single", +# ~~~~^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +# compileflags, True), test.globs) +# ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +# File "", line 1, in +# 1/0 +# ~^~ +# ZeroDivisionError: division by zero +# ---------------------------------------------------------------------- +# File "...sample_doctest_errors.py", line 23, in test.test_doctest.sample_doctest_errors.errors +# Failed example: +# f() +# Exception raised: +# Traceback (most recent call last): +# File ... +# exec(compile(example.source, filename, "single", +# ~~~~^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +# compileflags, True), test.globs) +# ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +# File "", line 1, in +# f() +# ~^^ +# File "", line 2, in f +# 2 + '2' +# ~~^~~~~ +# TypeError: ... +# ---------------------------------------------------------------------- +# File "...sample_doctest_errors.py", line 25, in test.test_doctest.sample_doctest_errors.errors +# Failed example: +# g() +# Exception raised: +# Traceback (most recent call last): +# File ... +# exec(compile(example.source, filename, "single", +# ~~~~^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +# compileflags, True), test.globs) +# ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +# File "", line 1, in +# g() +# ~^^ +# File "...sample_doctest_errors.py", line 12, in g +# [][0] # line 12 +# ~~^^^ +# IndexError: list index out of range +# +# +# >>> print(result.failures[3][1]) # doctest: +ELLIPSIS +# Traceback (most recent call last): +# File ... +# raise self.failureException(self.format_failure(new.getvalue())) +# AssertionError: Failed doctest test for test.test_doctest.sample_doctest_errors.syntax_error +# File "...sample_doctest_errors.py", line 29, in syntax_error +# +# ---------------------------------------------------------------------- +# File "...sample_doctest_errors.py", line 31, in test.test_doctest.sample_doctest_errors.syntax_error +# Failed example: +# 2+*3 +# Exception raised: +# Traceback (most recent call last): +# File ... +# exec(compile(example.source, filename, "single", +# ~~~~~~~^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +# compileflags, True), test.globs) +# ^^^^^^^^^^^^^^^^^^^ +# File "", line 1 +# 2+*3 +# ^ +# SyntaxError: invalid syntax +# +# +# """ + +# TODO: RUSTPYTHON +# try finally does not work in the interactive shell +# def test_DocFileSuite(): +# """We can test tests found in text files using a DocFileSuite. + +# We create a suite by providing the names of one or more text +# files that include examples: + +# >>> import unittest +# >>> suite = doctest.DocFileSuite('test_doctest.txt', +# ... 'test_doctest2.txt', +# ... 'test_doctest4.txt') +# >>> suite.run(unittest.TestResult()) +# + +# The test files are looked for in the directory containing the +# calling module. A package keyword argument can be provided to +# specify a different relative location. + +# >>> import unittest +# >>> suite = doctest.DocFileSuite('test_doctest.txt', +# ... 'test_doctest2.txt', +# ... 'test_doctest4.txt', +# ... package='test.test_doctest') +# >>> suite.run(unittest.TestResult()) +# + +# Support for using a package's __loader__.get_data() is also +# provided. + +# >>> import unittest, pkgutil, test +# >>> added_loader = False +# >>> if not hasattr(test, '__loader__'): +# ... test.__loader__ = pkgutil.get_loader(test) +# ... added_loader = True +# >>> try: +# ... suite = doctest.DocFileSuite('test_doctest.txt', +# ... 'test_doctest2.txt', +# ... 'test_doctest4.txt', +# ... package='test.test_doctest') +# ... suite.run(unittest.TestResult()) +# ... finally: +# ... if added_loader: +# ... del test.__loader__ +# + +# '/' should be used as a path separator. It will be converted +# to a native separator at run time: + +# >>> suite = doctest.DocFileSuite('../test_doctest/test_doctest.txt') +# >>> suite.run(unittest.TestResult()) +# + +# If DocFileSuite is used from an interactive session, then files +# are resolved relative to the directory of sys.argv[0]: + +# >>> import types, os.path +# >>> from test.test_doctest import test_doctest +# >>> save_argv = sys.argv +# >>> sys.argv = [test_doctest.__file__] +# >>> suite = doctest.DocFileSuite('test_doctest.txt', +# ... package=types.ModuleType('__main__')) +# >>> sys.argv = save_argv + +# By setting `module_relative=False`, os-specific paths may be +# used (including absolute paths and paths relative to the +# working directory): + +# >>> # Get the absolute path of the test package. +# >>> test_doctest_path = os.path.abspath(test_doctest.__file__) +# >>> test_pkg_path = os.path.split(test_doctest_path)[0] + +# >>> # Use it to find the absolute path of test_doctest.txt. +# >>> test_file = os.path.join(test_pkg_path, 'test_doctest.txt') + +# >>> suite = doctest.DocFileSuite(test_file, module_relative=False) +# >>> suite.run(unittest.TestResult()) +# + +# It is an error to specify `package` when `module_relative=False`: + +# >>> suite = doctest.DocFileSuite(test_file, module_relative=False, +# ... package='test') +# Traceback (most recent call last): +# ValueError: Package may only be specified for module-relative paths. + +# If all examples in a file are skipped, unittest will report it as a +# skipped test: + +# >>> suite = doctest.DocFileSuite('test_doctest.txt', +# ... 'test_doctest4.txt', +# ... 'test_doctest_skip.txt', +# ... 'test_doctest_skip2.txt') +# >>> result = suite.run(unittest.TestResult()) +# >>> result +# +# >>> len(result.skipped) +# 1 +# >>> for tst, _ in result.skipped: # doctest: +ELLIPSIS +# ... print('=', tst) +# = ...test_doctest_skip.txt + +# You can specify initial global variables: + +# >>> suite = doctest.DocFileSuite('test_doctest.txt', +# ... 'test_doctest2.txt', +# ... 'test_doctest4.txt', +# ... globs={'favorite_color': 'blue'}) +# >>> suite.run(unittest.TestResult()) +# + +# In this case, we supplied a missing favorite color. You can +# provide doctest options: + +# >>> suite = doctest.DocFileSuite('test_doctest.txt', +# ... 'test_doctest2.txt', +# ... 'test_doctest4.txt', +# ... optionflags=doctest.DONT_ACCEPT_BLANKLINE, +# ... globs={'favorite_color': 'blue'}) +# >>> suite.run(unittest.TestResult()) +# + +# And, you can provide setUp and tearDown functions: + +# >>> def setUp(t): +# ... from test.test_doctest import test_doctest +# ... test_doctest.sillySetup = True + +# >>> def tearDown(t): +# ... from test.test_doctest import test_doctest +# ... del test_doctest.sillySetup + +# Here, we installed a silly variable that the test expects: + +# >>> suite = doctest.DocFileSuite('test_doctest.txt', +# ... 'test_doctest2.txt', +# ... 'test_doctest4.txt', +# ... setUp=setUp, tearDown=tearDown) +# >>> suite.run(unittest.TestResult()) +# + +# But the tearDown restores sanity: + +# >>> from test.test_doctest import test_doctest +# >>> test_doctest.sillySetup +# Traceback (most recent call last): +# ... +# AttributeError: module 'test.test_doctest.test_doctest' has no attribute 'sillySetup' + +# The setUp and tearDown functions are passed test objects. +# Here, we'll use a setUp function to set the favorite color in +# test_doctest.txt: + +# >>> def setUp(test): +# ... test.globs['favorite_color'] = 'blue' + +# >>> suite = doctest.DocFileSuite('test_doctest.txt', setUp=setUp) +# >>> suite.run(unittest.TestResult()) +# + +# Here, we didn't need to use a tearDown function because we +# modified the test globals. The test globals are +# automatically cleared for us after a test. + +# Tests in a file run using `DocFileSuite` can also access the +# `__file__` global, which is set to the name of the file +# containing the tests: + +# >>> suite = doctest.DocFileSuite('test_doctest3.txt') +# >>> suite.run(unittest.TestResult()) +# + +# If the tests contain non-ASCII characters, we have to specify which +# encoding the file is encoded with. We do so by using the `encoding` +# parameter: + +# >>> suite = doctest.DocFileSuite('test_doctest.txt', +# ... 'test_doctest2.txt', +# ... 'test_doctest4.txt', +# ... encoding='utf-8') +# >>> suite.run(unittest.TestResult()) +# +# """ + +# TODO: RUSTPYTHON +# traceback + error message is different than in CPython +# def test_DocFileSuite_errors(): +# """Tests for error reporting in DocTestSuite. + +# >>> import unittest +# >>> suite = doctest.DocFileSuite('test_doctest_errors.txt') +# >>> result = suite.run(unittest.TestResult()) +# >>> result +# +# >>> print(result.failures[0][1]) # doctest: +ELLIPSIS +# Traceback (most recent call last): +# File ... +# raise self.failureException(self.format_failure(new.getvalue())) +# AssertionError: Failed doctest test for test_doctest_errors.txt +# File "...test_doctest_errors.txt", line 0 +# +# ---------------------------------------------------------------------- +# File "...test_doctest_errors.txt", line 4, in test_doctest_errors.txt +# Failed example: +# 2 + 2 +# Expected: +# 5 +# Got: +# 4 +# ---------------------------------------------------------------------- +# File "...test_doctest_errors.txt", line 6, in test_doctest_errors.txt +# Failed example: +# 1/0 +# Exception raised: +# Traceback (most recent call last): +# File ... +# exec(compile(example.source, filename, "single", +# ~~~~^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +# compileflags, True), test.globs) +# ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +# File "", line 1, in +# 1/0 +# ~^~ +# ZeroDivisionError: division by zero +# ---------------------------------------------------------------------- +# File "...test_doctest_errors.txt", line 11, in test_doctest_errors.txt +# Failed example: +# f() +# Exception raised: +# Traceback (most recent call last): +# File ... +# exec(compile(example.source, filename, "single", +# ~~~~^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +# compileflags, True), test.globs) +# ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +# File "", line 1, in +# f() +# ~^^ +# File "", line 2, in f +# 2 + '2' +# ~~^~~~~ +# TypeError: ... +# ---------------------------------------------------------------------- +# File "...test_doctest_errors.txt", line 13, in test_doctest_errors.txt +# Failed example: +# 2+*3 +# Exception raised: +# Traceback (most recent call last): +# File ... +# exec(compile(example.source, filename, "single", +# ~~~~~~~^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +# compileflags, True), test.globs) +# ^^^^^^^^^^^^^^^^^^^ +# File "", line 1 +# 2+*3 +# ^ +# SyntaxError: invalid syntax +# +# +# """ + +def test_trailing_space_in_test(): + """ + Trailing spaces in expected output are significant: + + >>> x, y = 'foo', '' + >>> print(x, y) + foo \n + """ + +class Wrapper: + def __init__(self, func): + self.func = func + functools.update_wrapper(self, func) + + def __call__(self, *args, **kwargs): + self.func(*args, **kwargs) + +@Wrapper +def wrapped(): + """ + Docstrings in wrapped functions must be detected as well. + + >>> 'one other test' + 'one other test' + """ + +def test_look_in_unwrapped(): + """ + Ensure that wrapped doctests work correctly. + + >>> import doctest + >>> doctest.run_docstring_examples( + ... wrapped, {}, name=wrapped.__name__, verbose=True) + Finding tests in wrapped + Trying: + 'one other test' + Expecting: + 'one other test' + ok + """ + +@doctest_skip_if(support.check_impl_detail(cpython=False)) +def test_wrapped_c_func(): + """ + # https://github.com/python/cpython/issues/117692 + >>> import binascii + >>> from test.test_doctest.decorator_mod import decorator + + >>> c_func_wrapped = decorator(binascii.b2a_hex) + >>> tests = doctest.DocTestFinder(exclude_empty=False).find(c_func_wrapped) + >>> for test in tests: + ... print(test.lineno, test.name) + None b2a_hex + """ + +def test_unittest_reportflags(): + """Default unittest reporting flags can be set to control reporting + + Here, we'll set the REPORT_ONLY_FIRST_FAILURE option so we see + only the first failure of each test. First, we'll look at the + output without the flag. The file test_doctest.txt file has two + tests. They both fail if blank lines are disabled: + + >>> suite = doctest.DocFileSuite('test_doctest.txt', + ... optionflags=doctest.DONT_ACCEPT_BLANKLINE) + >>> import unittest + >>> result = suite.run(unittest.TestResult()) + >>> result + + >>> print(result.failures[0][1]) # doctest: +ELLIPSIS + Traceback ... + Failed example: + favorite_color + ... + Failed example: + if 1: + ... + + Note that we see both failures displayed. + + >>> old = doctest.set_unittest_reportflags( + ... doctest.REPORT_ONLY_FIRST_FAILURE) + + Now, when we run the test: + + >>> result = suite.run(unittest.TestResult()) + >>> result + + >>> print(result.failures[0][1]) # doctest: +ELLIPSIS + Traceback ... + Failed example: + favorite_color + Exception raised: + ... + NameError: name 'favorite_color' is not defined + + + + We get only the first failure. + + If we give any reporting options when we set up the tests, + however: + + >>> suite = doctest.DocFileSuite('test_doctest.txt', + ... optionflags=doctest.DONT_ACCEPT_BLANKLINE | doctest.REPORT_NDIFF) + + Then the default eporting options are ignored: + + >>> result = suite.run(unittest.TestResult()) + >>> result + + + *NOTE*: These doctest are intentionally not placed in raw string to depict + the trailing whitespace using `\x20` in the diff below. + + >>> print(result.failures[0][1]) # doctest: +ELLIPSIS + Traceback ... + Failed example: + favorite_color + ... + Failed example: + if 1: + print('a') + print() + print('b') + Differences (ndiff with -expected +actual): + a + - + +\x20 + b + + + + + Test runners can restore the formatting flags after they run: + + >>> ignored = doctest.set_unittest_reportflags(old) + + """ + +def test_testfile(): r""" +Tests for the `testfile()` function. This function runs all the +doctest examples in a given file. In its simple invocation, it is +called with the name of a file, which is taken to be relative to the +calling module. The return value is (#failures, #tests). + +We don't want color or `-v` in sys.argv for these tests. + + >>> save_colorize = _colorize.COLORIZE + >>> _colorize.COLORIZE = False + + >>> save_argv = sys.argv + >>> if '-v' in sys.argv: + ... sys.argv = [arg for arg in save_argv if arg != '-v'] + + + >>> doctest.testfile('test_doctest.txt') # doctest: +ELLIPSIS + ********************************************************************** + File "...", line 6, in test_doctest.txt + Failed example: + favorite_color + Exception raised: + ... + NameError: name 'favorite_color' is not defined + ********************************************************************** + 1 item had failures: + 1 of 2 in test_doctest.txt + ***Test Failed*** 1 failure. + TestResults(failed=1, attempted=2) + >>> doctest.master = None # Reset master. + +(Note: we'll be clearing doctest.master after each call to +`doctest.testfile`, to suppress warnings about multiple tests with the +same name.) + +Globals may be specified with the `globs` and `extraglobs` parameters: + + >>> globs = {'favorite_color': 'blue'} + >>> doctest.testfile('test_doctest.txt', globs=globs) + TestResults(failed=0, attempted=2) + >>> doctest.master = None # Reset master. + + >>> extraglobs = {'favorite_color': 'red'} + >>> doctest.testfile('test_doctest.txt', globs=globs, + ... extraglobs=extraglobs) # doctest: +ELLIPSIS + ********************************************************************** + File "...", line 6, in test_doctest.txt + Failed example: + favorite_color + Expected: + 'blue' + Got: + 'red' + ********************************************************************** + 1 item had failures: + 1 of 2 in test_doctest.txt + ***Test Failed*** 1 failure. + TestResults(failed=1, attempted=2) + >>> doctest.master = None # Reset master. + +The file may be made relative to a given module or package, using the +optional `module_relative` parameter: + + >>> doctest.testfile('test_doctest.txt', globs=globs, + ... module_relative='test') + TestResults(failed=0, attempted=2) + >>> doctest.master = None # Reset master. + +Verbosity can be increased with the optional `verbose` parameter: + + >>> doctest.testfile('test_doctest.txt', globs=globs, verbose=True) + Trying: + favorite_color + Expecting: + 'blue' + ok + Trying: + if 1: + print('a') + print() + print('b') + Expecting: + a + + b + ok + 1 item passed all tests: + 2 tests in test_doctest.txt + 2 tests in 1 item. + 2 passed. + Test passed. + TestResults(failed=0, attempted=2) + >>> doctest.master = None # Reset master. + +The name of the test may be specified with the optional `name` +parameter: + + >>> doctest.testfile('test_doctest.txt', name='newname') + ... # doctest: +ELLIPSIS + ********************************************************************** + File "...", line 6, in newname + ... + TestResults(failed=1, attempted=2) + >>> doctest.master = None # Reset master. + +The summary report may be suppressed with the optional `report` +parameter: + + >>> doctest.testfile('test_doctest.txt', report=False) + ... # doctest: +ELLIPSIS + ********************************************************************** + File "...", line 6, in test_doctest.txt + Failed example: + favorite_color + Exception raised: + ... + NameError: name 'favorite_color' is not defined + TestResults(failed=1, attempted=2) + >>> doctest.master = None # Reset master. + +The optional keyword argument `raise_on_error` can be used to raise an +exception on the first error (which may be useful for postmortem +debugging): + + >>> doctest.testfile('test_doctest.txt', raise_on_error=True) + ... # doctest: +ELLIPSIS + Traceback (most recent call last): + doctest.UnexpectedException: ... + >>> doctest.master = None # Reset master. + +If the tests contain non-ASCII characters, the tests might fail, since +it's unknown which encoding is used. The encoding can be specified +using the optional keyword argument `encoding`: + + >>> doctest.testfile('test_doctest4.txt', encoding='latin-1') # doctest: +ELLIPSIS + ********************************************************************** + File "...", line 7, in test_doctest4.txt + Failed example: + '...' + Expected: + 'f\xf6\xf6' + Got: + 'f\xc3\xb6\xc3\xb6' + ********************************************************************** + ... + ********************************************************************** + 1 item had failures: + 2 of 2 in test_doctest4.txt + ***Test Failed*** 2 failures. + TestResults(failed=2, attempted=2) + >>> doctest.master = None # Reset master. + + >>> doctest.testfile('test_doctest4.txt', encoding='utf-8') + TestResults(failed=0, attempted=2) + >>> doctest.master = None # Reset master. + +Test the verbose output: + + >>> doctest.testfile('test_doctest4.txt', encoding='utf-8', verbose=True) + Trying: + 'föö' + Expecting: + 'f\xf6\xf6' + ok + Trying: + 'bÄ…r' + Expecting: + 'b\u0105r' + ok + 1 item passed all tests: + 2 tests in test_doctest4.txt + 2 tests in 1 item. + 2 passed. + Test passed. + TestResults(failed=0, attempted=2) + >>> doctest.master = None # Reset master. + >>> sys.argv = save_argv + >>> _colorize.COLORIZE = save_colorize +""" + +# TODO: RUSTPYTHON +# traceback + error message is different than in CPython +# def test_testfile_errors(): r""" +# Tests for error reporting in the testfile() function. + +# >>> doctest.testfile('test_doctest_errors.txt', verbose=False) # doctest: +ELLIPSIS +# ********************************************************************** +# File "...test_doctest_errors.txt", line 4, in test_doctest_errors.txt +# Failed example: +# 2 + 2 +# Expected: +# 5 +# Got: +# 4 +# ********************************************************************** +# File "...test_doctest_errors.txt", line 6, in test_doctest_errors.txt +# Failed example: +# 1/0 +# Exception raised: +# Traceback (most recent call last): +# File ... +# exec(compile(example.source, filename, "single", +# ~~~~^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +# compileflags, True), test.globs) +# ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +# File "", line 1, in +# 1/0 +# ~^~ +# ZeroDivisionError: division by zero +# ********************************************************************** +# File "...test_doctest_errors.txt", line 11, in test_doctest_errors.txt +# Failed example: +# f() +# Exception raised: +# Traceback (most recent call last): +# File ... +# exec(compile(example.source, filename, "single", +# ~~~~^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +# compileflags, True), test.globs) +# ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +# File "", line 1, in +# f() +# ~^^ +# File "", line 2, in f +# 2 + '2' +# ~~^~~~~ +# TypeError: ... +# ********************************************************************** +# File "...test_doctest_errors.txt", line 13, in test_doctest_errors.txt +# Failed example: +# 2+*3 +# Exception raised: +# Traceback (most recent call last): +# File ... +# exec(compile(example.source, filename, "single", +# ~~~~~~~^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +# compileflags, True), test.globs) +# ^^^^^^^^^^^^^^^^^^^ +# File "", line 1 +# 2+*3 +# ^ +# SyntaxError: invalid syntax +# ********************************************************************** +# 1 item had failures: +# 4 of 5 in test_doctest_errors.txt +# ***Test Failed*** 4 failures. +# TestResults(failed=4, attempted=5) +# """ + +class TestImporter(importlib.abc.MetaPathFinder): + + def find_spec(self, fullname, path, target=None): + return importlib.util.spec_from_file_location(fullname, path, loader=self) + + def get_data(self, path): + with open(path, mode='rb') as f: + return f.read() + + def exec_module(self, module): + raise ImportError + + def create_module(self, spec): + return None + +class TestHook: + + def __init__(self, pathdir): + self.sys_path = sys.path[:] + self.meta_path = sys.meta_path[:] + self.path_hooks = sys.path_hooks[:] + sys.path.append(pathdir) + sys.path_importer_cache.clear() + self.modules_before = sys.modules.copy() + self.importer = TestImporter() + sys.meta_path.append(self.importer) + + def remove(self): + sys.path[:] = self.sys_path + sys.meta_path[:] = self.meta_path + sys.path_hooks[:] = self.path_hooks + sys.path_importer_cache.clear() + sys.modules.clear() + sys.modules.update(self.modules_before) + + +@contextlib.contextmanager +def test_hook(pathdir): + hook = TestHook(pathdir) + try: + yield hook + finally: + hook.remove() + +# TODO: RUSTPYTHON +# f.write(b'Test:\r\n\r\n >>> x = 1 + 1\r\n\r\nDone.\r\n') +# does not return in with... +# def test_lineendings(): r""" +# *nix systems use \n line endings, while Windows systems use \r\n, and +# old Mac systems used \r, which Python still recognizes as a line ending. Python +# handles this using universal newline mode for reading files. Let's make +# sure doctest does so (issue 8473) by creating temporary test files using each +# of the three line disciplines. At least one will not match either the universal +# newline \n or os.linesep for the platform the test is run on. + +# Windows line endings first: + +# >>> import tempfile, os +# >>> fn = tempfile.mktemp() +# >>> with open(fn, 'wb') as f: +# ... f.write(b'Test:\r\n\r\n >>> x = 1 + 1\r\n\r\nDone.\r\n') +# 35 +# >>> doctest.testfile(fn, module_relative=False, verbose=False) +# TestResults(failed=0, attempted=1) +# >>> os.remove(fn) + +# And now *nix line endings: + +# >>> fn = tempfile.mktemp() +# >>> with open(fn, 'wb') as f: +# ... f.write(b'Test:\n\n >>> x = 1 + 1\n\nDone.\n') +# 30 +# >>> doctest.testfile(fn, module_relative=False, verbose=False) +# TestResults(failed=0, attempted=1) +# >>> os.remove(fn) + +# And finally old Mac line endings: + +# >>> fn = tempfile.mktemp() +# >>> with open(fn, 'wb') as f: +# ... f.write(b'Test:\r\r >>> x = 1 + 1\r\rDone.\r') +# 30 +# >>> doctest.testfile(fn, module_relative=False, verbose=False) +# TestResults(failed=0, attempted=1) +# >>> os.remove(fn) + +# Now we test with a package loader that has a get_data method, since that +# bypasses the standard universal newline handling so doctest has to do the +# newline conversion itself; let's make sure it does so correctly (issue 1812). +# We'll write a file inside the package that has all three kinds of line endings +# in it, and use a package hook to install a custom loader; on any platform, +# at least one of the line endings will raise a ValueError for inconsistent +# whitespace if doctest does not correctly do the newline conversion. + +# >>> from test.support import os_helper +# >>> import shutil +# >>> dn = tempfile.mkdtemp() +# >>> pkg = os.path.join(dn, "doctest_testpkg") +# >>> os.mkdir(pkg) +# >>> os_helper.create_empty_file(os.path.join(pkg, "__init__.py")) +# >>> fn = os.path.join(pkg, "doctest_testfile.txt") +# >>> with open(fn, 'wb') as f: +# ... f.write( +# ... b'Test:\r\n\r\n' +# ... b' >>> x = 1 + 1\r\n\r\n' +# ... b'Done.\r\n' +# ... b'Test:\n\n' +# ... b' >>> x = 1 + 1\n\n' +# ... b'Done.\n' +# ... b'Test:\r\r' +# ... b' >>> x = 1 + 1\r\r' +# ... b'Done.\r' +# ... ) +# 95 +# >>> with test_hook(dn): +# ... doctest.testfile("doctest_testfile.txt", package="doctest_testpkg", verbose=False) +# TestResults(failed=0, attempted=3) +# >>> shutil.rmtree(dn) + +# """ + +def test_testmod(): r""" +Tests for the testmod function. More might be useful, but for now we're just +testing the case raised by Issue 6195, where trying to doctest a C module would +fail with a UnicodeDecodeError because doctest tried to read the "source" lines +out of the binary module. + + >>> import unicodedata + >>> doctest.testmod(unicodedata, verbose=False) + TestResults(failed=0, attempted=0) +""" + +# TODO: RUSTPYTHON +# traceback + error message is different than in CPython +# def test_testmod_errors(): r""" +# Tests for error reporting in the testmod() function. + +# >>> import test.test_doctest.sample_doctest_errors as mod +# >>> doctest.testmod(mod, verbose=False) # doctest: +ELLIPSIS +# ********************************************************************** +# File "...sample_doctest_errors.py", line 5, in test.test_doctest.sample_doctest_errors +# Failed example: +# 2 + 2 +# Expected: +# 5 +# Got: +# 4 +# ********************************************************************** +# File "...sample_doctest_errors.py", line 7, in test.test_doctest.sample_doctest_errors +# Failed example: +# 1/0 +# Exception raised: +# Traceback (most recent call last): +# File ... +# exec(compile(example.source, filename, "single", +# ~~~~^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +# compileflags, True), test.globs) +# ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +# File "", line 1, in +# 1/0 +# ~^~ +# ZeroDivisionError: division by zero +# ********************************************************************** +# File "...sample_doctest_errors.py", line ?, in test.test_doctest.sample_doctest_errors.__test__.bad +# Failed example: +# 2 + 2 +# Expected: +# 5 +# Got: +# 4 +# ********************************************************************** +# File "...sample_doctest_errors.py", line ?, in test.test_doctest.sample_doctest_errors.__test__.bad +# Failed example: +# 1/0 +# Exception raised: +# Traceback (most recent call last): +# File ... +# exec(compile(example.source, filename, "single", +# ~~~~^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +# compileflags, True), test.globs) +# ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +# File "", line 1, in +# 1/0 +# ~^~ +# ZeroDivisionError: division by zero +# ********************************************************************** +# File "...sample_doctest_errors.py", line 16, in test.test_doctest.sample_doctest_errors.errors +# Failed example: +# 2 + 2 +# Expected: +# 5 +# Got: +# 4 +# ********************************************************************** +# File "...sample_doctest_errors.py", line 18, in test.test_doctest.sample_doctest_errors.errors +# Failed example: +# 1/0 +# Exception raised: +# Traceback (most recent call last): +# File ... +# exec(compile(example.source, filename, "single", +# ~~~~^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +# compileflags, True), test.globs) +# ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +# File "", line 1, in +# 1/0 +# ~^~ +# ZeroDivisionError: division by zero +# ********************************************************************** +# File "...sample_doctest_errors.py", line 23, in test.test_doctest.sample_doctest_errors.errors +# Failed example: +# f() +# Exception raised: +# Traceback (most recent call last): +# File ... +# exec(compile(example.source, filename, "single", +# ~~~~^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +# compileflags, True), test.globs) +# ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +# File "", line 1, in +# f() +# ~^^ +# File "", line 2, in f +# 2 + '2' +# ~~^~~~~ +# TypeError: ... +# ********************************************************************** +# File "...sample_doctest_errors.py", line 25, in test.test_doctest.sample_doctest_errors.errors +# Failed example: +# g() +# Exception raised: +# Traceback (most recent call last): +# File ... +# exec(compile(example.source, filename, "single", +# ~~~~^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +# compileflags, True), test.globs) +# ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +# File "", line 1, in +# g() +# ~^^ +# File "...sample_doctest_errors.py", line 12, in g +# [][0] # line 12 +# ~~^^^ +# IndexError: list index out of range +# ********************************************************************** +# File "...sample_doctest_errors.py", line 31, in test.test_doctest.sample_doctest_errors.syntax_error +# Failed example: +# 2+*3 +# Exception raised: +# Traceback (most recent call last): +# File ... +# exec(compile(example.source, filename, "single", +# ~~~~~~~^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +# compileflags, True), test.globs) +# ^^^^^^^^^^^^^^^^^^^ +# File "", line 1 +# 2+*3 +# ^ +# SyntaxError: invalid syntax +# ********************************************************************** +# 4 items had failures: +# 2 of 2 in test.test_doctest.sample_doctest_errors +# 2 of 2 in test.test_doctest.sample_doctest_errors.__test__.bad +# 4 of 5 in test.test_doctest.sample_doctest_errors.errors +# 1 of 1 in test.test_doctest.sample_doctest_errors.syntax_error +# ***Test Failed*** 9 failures. +# TestResults(failed=9, attempted=10) +# """ + +try: + os.fsencode("foo-bär@baz.py") + supports_unicode = True +except UnicodeEncodeError: + # Skip the test: the filesystem encoding is unable to encode the filename + supports_unicode = False + +# TODO: RUSTPYTHON +# traceback message is different than in CPython +# if supports_unicode: +# def test_unicode(): """ +# Check doctest with a non-ascii filename: + +# >>> save_colorize = _colorize.COLORIZE +# >>> _colorize.COLORIZE = False + +# >>> doc = ''' +# ... >>> raise Exception('clé') +# ... ''' +# ... +# >>> parser = doctest.DocTestParser() +# >>> test = parser.get_doctest(doc, {}, "foo-bär@baz", "foo-bär@baz.py", 0) +# >>> test +# +# >>> runner = doctest.DocTestRunner(verbose=False) +# >>> runner.run(test) # doctest: +ELLIPSIS +# ********************************************************************** +# File "foo-bär@baz.py", line 2, in foo-bär@baz +# Failed example: +# raise Exception('clé') +# Exception raised: +# Traceback (most recent call last): +# File ... +# exec(compile(example.source, filename, "single", +# ~~~~^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +# compileflags, True), test.globs) +# ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +# File "", line 1, in +# raise Exception('clé') +# Exception: clé +# TestResults(failed=1, attempted=1) + +# >>> _colorize.COLORIZE = save_colorize +# """ + + +# TODO: RUSTPYTHON +# Difference in Error: +# FileNotFoundError: [Errno ...] ...nosuchfile... vs FileNotFoundError: (2, 'No such file or directory') +# @doctest_skip_if(not support.has_subprocess_support) +# def test_CLI(): r""" +# The doctest module can be used to run doctests against an arbitrary file. +# These tests test this CLI functionality. + +# We'll use the support module's script_helpers for this, and write a test files +# to a temp dir to run the command against. Due to a current limitation in +# script_helpers, though, we need a little utility function to turn the returned +# output into something we can doctest against: + +# >>> def normalize(s): +# ... return '\n'.join(s.decode().splitlines()) + +# With those preliminaries out of the way, we'll start with a file with two +# simple tests and no errors. We'll run both the unadorned doctest command, and +# the verbose version, and then check the output: + +# >>> from test.support import script_helper +# >>> from test.support.os_helper import temp_dir +# >>> with temp_dir() as tmpdir: +# ... fn = os.path.join(tmpdir, 'myfile.doc') +# ... with open(fn, 'w', encoding='utf-8') as f: +# ... _ = f.write('This is a very simple test file.\n') +# ... _ = f.write(' >>> 1 + 1\n') +# ... _ = f.write(' 2\n') +# ... _ = f.write(' >>> "a"\n') +# ... _ = f.write(" 'a'\n") +# ... _ = f.write('\n') +# ... _ = f.write('And that is it.\n') +# ... rc1, out1, err1 = script_helper.assert_python_ok( +# ... '-m', 'doctest', fn) +# ... rc2, out2, err2 = script_helper.assert_python_ok( +# ... '-m', 'doctest', '-v', fn) + +# With no arguments and passing tests, we should get no output: + +# >>> rc1, out1, err1 +# (0, b'', b'') + +# With the verbose flag, we should see the test output, but no error output: + +# >>> rc2, err2 +# (0, b'') +# >>> print(normalize(out2)) +# Trying: +# 1 + 1 +# Expecting: +# 2 +# ok +# Trying: +# "a" +# Expecting: +# 'a' +# ok +# 1 item passed all tests: +# 2 tests in myfile.doc +# 2 tests in 1 item. +# 2 passed. +# Test passed. + +# Now we'll write a couple files, one with three tests, the other a python module +# with two tests, both of the files having "errors" in the tests that can be made +# non-errors by applying the appropriate doctest options to the run (ELLIPSIS in +# the first file, NORMALIZE_WHITESPACE in the second). This combination will +# allow thoroughly testing the -f and -o flags, as well as the doctest command's +# ability to process more than one file on the command line and, since the second +# file ends in '.py', its handling of python module files (as opposed to straight +# text files). + +# >>> from test.support import script_helper +# >>> from test.support.os_helper import temp_dir +# >>> with temp_dir() as tmpdir: +# ... fn = os.path.join(tmpdir, 'myfile.doc') +# ... with open(fn, 'w', encoding="utf-8") as f: +# ... _ = f.write('This is another simple test file.\n') +# ... _ = f.write(' >>> 1 + 1\n') +# ... _ = f.write(' 2\n') +# ... _ = f.write(' >>> "abcdef"\n') +# ... _ = f.write(" 'a...f'\n") +# ... _ = f.write(' >>> "ajkml"\n') +# ... _ = f.write(" 'a...l'\n") +# ... _ = f.write('\n') +# ... _ = f.write('And that is it.\n') +# ... fn2 = os.path.join(tmpdir, 'myfile2.py') +# ... with open(fn2, 'w', encoding='utf-8') as f: +# ... _ = f.write('def test_func():\n') +# ... _ = f.write(' \"\"\"\n') +# ... _ = f.write(' This is simple python test function.\n') +# ... _ = f.write(' >>> 1 + 1\n') +# ... _ = f.write(' 2\n') +# ... _ = f.write(' >>> "abc def"\n') +# ... _ = f.write(" 'abc def'\n") +# ... _ = f.write("\n") +# ... _ = f.write(' \"\"\"\n') +# ... rc1, out1, err1 = script_helper.assert_python_failure( +# ... '-m', 'doctest', fn, fn2) +# ... rc2, out2, err2 = script_helper.assert_python_ok( +# ... '-m', 'doctest', '-o', 'ELLIPSIS', fn) +# ... rc3, out3, err3 = script_helper.assert_python_ok( +# ... '-m', 'doctest', '-o', 'ELLIPSIS', +# ... '-o', 'NORMALIZE_WHITESPACE', fn, fn2) +# ... rc4, out4, err4 = script_helper.assert_python_failure( +# ... '-m', 'doctest', '-f', fn, fn2) +# ... rc5, out5, err5 = script_helper.assert_python_ok( +# ... '-m', 'doctest', '-v', '-o', 'ELLIPSIS', +# ... '-o', 'NORMALIZE_WHITESPACE', fn, fn2) + +# Our first test run will show the errors from the first file (doctest stops if a +# file has errors). Note that doctest test-run error output appears on stdout, +# not stderr: + +# >>> rc1, err1 +# (1, b'') +# >>> print(normalize(out1)) # doctest: +ELLIPSIS +# ********************************************************************** +# File "...myfile.doc", line 4, in myfile.doc +# Failed example: +# "abcdef" +# Expected: +# 'a...f' +# Got: +# 'abcdef' +# ********************************************************************** +# File "...myfile.doc", line 6, in myfile.doc +# Failed example: +# "ajkml" +# Expected: +# 'a...l' +# Got: +# 'ajkml' +# ********************************************************************** +# 1 item had failures: +# 2 of 3 in myfile.doc +# ***Test Failed*** 2 failures. + +# With -o ELLIPSIS specified, the second run, against just the first file, should +# produce no errors, and with -o NORMALIZE_WHITESPACE also specified, neither +# should the third, which ran against both files: + +# >>> rc2, out2, err2 +# (0, b'', b'') +# >>> rc3, out3, err3 +# (0, b'', b'') + +# The fourth run uses FAIL_FAST, so we should see only one error: + +# >>> rc4, err4 +# (1, b'') +# >>> print(normalize(out4)) # doctest: +ELLIPSIS +# ********************************************************************** +# File "...myfile.doc", line 4, in myfile.doc +# Failed example: +# "abcdef" +# Expected: +# 'a...f' +# Got: +# 'abcdef' +# ********************************************************************** +# 1 item had failures: +# 1 of 2 in myfile.doc +# ***Test Failed*** 1 failure. + +# The fifth test uses verbose with the two options, so we should get verbose +# success output for the tests in both files: + +# >>> rc5, err5 +# (0, b'') +# >>> print(normalize(out5)) +# Trying: +# 1 + 1 +# Expecting: +# 2 +# ok +# Trying: +# "abcdef" +# Expecting: +# 'a...f' +# ok +# Trying: +# "ajkml" +# Expecting: +# 'a...l' +# ok +# 1 item passed all tests: +# 3 tests in myfile.doc +# 3 tests in 1 item. +# 3 passed. +# Test passed. +# Trying: +# 1 + 1 +# Expecting: +# 2 +# ok +# Trying: +# "abc def" +# Expecting: +# 'abc def' +# ok +# 1 item had no tests: +# myfile2 +# 1 item passed all tests: +# 2 tests in myfile2.test_func +# 2 tests in 2 items. +# 2 passed. +# Test passed. + +# We should also check some typical error cases. + +# Invalid file name: + +# >>> rc, out, err = script_helper.assert_python_failure( +# ... '-m', 'doctest', 'nosuchfile') +# >>> rc, out +# (1, b'') +# >>> # The exact error message changes depending on the platform. +# >>> print(normalize(err)) # doctest: +ELLIPSIS +# Traceback (most recent call last): +# ... +# FileNotFoundError: [Errno ...] ...nosuchfile... + +# Invalid doctest option: + +# >>> rc, out, err = script_helper.assert_python_failure( +# ... '-m', 'doctest', '-o', 'nosuchoption') +# >>> rc, out +# (2, b'') +# >>> print(normalize(err)) # doctest: +ELLIPSIS +# usage...invalid...nosuchoption... + +# """ + +def test_no_trailing_whitespace_stripping(): + r""" + The fancy reports had a bug for a long time where any trailing whitespace on + the reported diff lines was stripped, making it impossible to see the + differences in line reported as different that differed only in the amount of + trailing whitespace. The whitespace still isn't particularly visible unless + you use NDIFF, but at least it is now there to be found. + + *NOTE*: This snippet was intentionally put inside a raw string to get rid of + leading whitespace error in executing the example below + + >>> def f(x): + ... r''' + ... >>> print('\n'.join(['a ', 'b'])) + ... a + ... b + ... ''' + """ + """ + *NOTE*: These doctest are not placed in raw string to depict the trailing whitespace + using `\x20` + + >>> test = doctest.DocTestFinder().find(f)[0] + >>> flags = doctest.REPORT_NDIFF + >>> doctest.DocTestRunner(verbose=False, optionflags=flags).run(test) + ... # doctest: +ELLIPSIS + ********************************************************************** + File ..., line 3, in f + Failed example: + print('\n'.join(['a ', 'b'])) + Differences (ndiff with -expected +actual): + - a + + a + b + TestResults(failed=1, attempted=1) + + *NOTE*: `\x20` is for checking the trailing whitespace on the +a line above. + We cannot use actual spaces there, as a commit hook prevents from committing + patches that contain trailing whitespace. More info on Issue 24746. + """ + + +def test_run_doctestsuite_multiple_times(): + """ + It was not possible to run the same DocTestSuite multiple times + http://bugs.python.org/issue2604 + http://bugs.python.org/issue9736 + + >>> import unittest + >>> import test.test_doctest.sample_doctest + >>> suite = doctest.DocTestSuite(test.test_doctest.sample_doctest) + >>> suite.run(unittest.TestResult()) + + >>> suite.run(unittest.TestResult()) + + """ + + +def test_exception_with_note(note): + """ + >>> save_colorize = _colorize.COLORIZE + >>> _colorize.COLORIZE = False + + >>> test_exception_with_note('Note') + Traceback (most recent call last): + ... + ValueError: Text + Note + + >>> test_exception_with_note('Note') # doctest: +IGNORE_EXCEPTION_DETAIL + Traceback (most recent call last): + ... + ValueError: Text + Note + + >>> test_exception_with_note('''Note + ... multiline + ... example''') + Traceback (most recent call last): + ValueError: Text + Note + multiline + example + + Different note will fail the test: + + >>> def f(x): + ... r''' + ... >>> exc = ValueError('message') + ... >>> exc.add_note('note') + ... >>> raise exc + ... Traceback (most recent call last): + ... ValueError: message + ... wrong note + ... ''' + >>> test = doctest.DocTestFinder().find(f)[0] + >>> doctest.DocTestRunner(verbose=False).run(test) + ... # doctest: +ELLIPSIS + ********************************************************************** + File "...", line 5, in f + Failed example: + raise exc + Expected: + Traceback (most recent call last): + ValueError: message + wrong note + Got: + Traceback (most recent call last): + ... + ValueError: message + note + TestResults(failed=1, attempted=...) + + >>> _colorize.COLORIZE = save_colorize + """ + exc = ValueError('Text') + exc.add_note(note) + raise exc + + +def test_exception_with_multiple_notes(): + """ + >>> test_exception_with_multiple_notes() + Traceback (most recent call last): + ... + ValueError: Text + One + Two + """ + exc = ValueError('Text') + exc.add_note('One') + exc.add_note('Two') + raise exc + + +def test_syntax_error_with_note(cls, multiline=False): + """ + >>> test_syntax_error_with_note(SyntaxError) + Traceback (most recent call last): + ... + SyntaxError: error + Note + + >>> test_syntax_error_with_note(SyntaxError) + Traceback (most recent call last): + SyntaxError: error + Note + + >>> test_syntax_error_with_note(SyntaxError) + Traceback (most recent call last): + ... + File "x.py", line 23 + bad syntax + SyntaxError: error + Note + + >>> test_syntax_error_with_note(IndentationError) + Traceback (most recent call last): + ... + IndentationError: error + Note + + >>> test_syntax_error_with_note(TabError, multiline=True) + Traceback (most recent call last): + ... + TabError: error + Note + Line + """ + exc = cls("error", ("x.py", 23, None, "bad syntax")) + exc.add_note('Note\nLine' if multiline else 'Note') + raise exc + + +def test_syntax_error_subclass_from_stdlib(): + """ + `ParseError` is a subclass of `SyntaxError`, but it is not a builtin: + + >>> test_syntax_error_subclass_from_stdlib() + Traceback (most recent call last): + ... + xml.etree.ElementTree.ParseError: error + error + Note + Line + """ + from xml.etree.ElementTree import ParseError + exc = ParseError("error\nerror") + exc.add_note('Note\nLine') + raise exc + + +def test_syntax_error_with_incorrect_expected_note(): + """ + >>> save_colorize = _colorize.COLORIZE + >>> _colorize.COLORIZE = False + + >>> def f(x): + ... r''' + ... >>> exc = SyntaxError("error", ("x.py", 23, None, "bad syntax")) + ... >>> exc.add_note('note1') + ... >>> exc.add_note('note2') + ... >>> raise exc + ... Traceback (most recent call last): + ... SyntaxError: error + ... wrong note + ... ''' + >>> test = doctest.DocTestFinder().find(f)[0] + >>> doctest.DocTestRunner(verbose=False).run(test) + ... # doctest: +ELLIPSIS + ********************************************************************** + File "...", line 6, in f + Failed example: + raise exc + Expected: + Traceback (most recent call last): + SyntaxError: error + wrong note + Got: + Traceback (most recent call last): + ... + SyntaxError: error + note1 + note2 + TestResults(failed=1, attempted=...) + + >>> _colorize.COLORIZE = save_colorize + """ + + +def load_tests(loader, tests, pattern): + tests.addTest(doctest.DocTestSuite(doctest)) + tests.addTest(doctest.DocTestSuite()) + return tests + + +if __name__ == '__main__': + unittest.main(module='test.test_doctest.test_doctest') diff --git a/Lib/test/test_doctest/test_doctest.txt b/Lib/test/test_doctest/test_doctest.txt new file mode 100644 index 00000000000..23446d1d224 --- /dev/null +++ b/Lib/test/test_doctest/test_doctest.txt @@ -0,0 +1,17 @@ +This is a sample doctest in a text file. + +In this example, we'll rely on a global variable being set for us +already: + + >>> favorite_color + 'blue' + +We can make this fail by disabling the blank-line feature. + + >>> if 1: + ... print('a') + ... print() + ... print('b') + a + + b diff --git a/Lib/test/test_doctest/test_doctest2.py b/Lib/test/test_doctest/test_doctest2.py new file mode 100644 index 00000000000..ab8a0696736 --- /dev/null +++ b/Lib/test/test_doctest/test_doctest2.py @@ -0,0 +1,126 @@ +"""A module to test whether doctest recognizes some 2.2 features, +like static and class methods. + +>>> print('yup') # 1 +yup + +We include some (random) encoded (utf-8) text in the text surrounding +the example. It should be ignored: + +ЉЊЈÐЂ + +""" + +import sys +import unittest +if sys.flags.optimize >= 2: + raise unittest.SkipTest("Cannot test docstrings with -O2") + +class C(object): + """Class C. + + >>> print(C()) # 2 + 42 + + + We include some (random) encoded (utf-8) text in the text surrounding + the example. It should be ignored: + + ЉЊЈÐЂ + + """ + + def __init__(self): + """C.__init__. + + >>> print(C()) # 3 + 42 + """ + + def __str__(self): + """ + >>> print(C()) # 4 + 42 + """ + return "42" + + class D(object): + """A nested D class. + + >>> print("In D!") # 5 + In D! + """ + + def nested(self): + """ + >>> print(3) # 6 + 3 + """ + + def getx(self): + """ + >>> c = C() # 7 + >>> c.x = 12 # 8 + >>> print(c.x) # 9 + -12 + """ + return -self._x + + def setx(self, value): + """ + >>> c = C() # 10 + >>> c.x = 12 # 11 + >>> print(c.x) # 12 + -12 + """ + self._x = value + + x = property(getx, setx, doc="""\ + >>> c = C() # 13 + >>> c.x = 12 # 14 + >>> print(c.x) # 15 + -12 + """) + + @staticmethod + def statm(): + """ + A static method. + + >>> print(C.statm()) # 16 + 666 + >>> print(C().statm()) # 17 + 666 + """ + return 666 + + @classmethod + def clsm(cls, val): + """ + A class method. + + >>> print(C.clsm(22)) # 18 + 22 + >>> print(C().clsm(23)) # 19 + 23 + """ + return val + + +class Test(unittest.TestCase): + def test_testmod(self): + import doctest, sys + EXPECTED = 19 + f, t = doctest.testmod(sys.modules[__name__]) + if f: + self.fail("%d of %d doctests failed" % (f, t)) + if t != EXPECTED: + self.fail("expected %d tests to run, not %d" % (EXPECTED, t)) + + +# Pollute the namespace with a bunch of imported functions and classes, +# to make sure they don't get tested. +from doctest import * + +if __name__ == '__main__': + unittest.main() diff --git a/Lib/test/test_doctest/test_doctest2.txt b/Lib/test/test_doctest/test_doctest2.txt new file mode 100644 index 00000000000..76dab94a9c0 --- /dev/null +++ b/Lib/test/test_doctest/test_doctest2.txt @@ -0,0 +1,14 @@ +This is a sample doctest in a text file. + +In this example, we'll rely on some silly setup: + + >>> import test.test_doctest.test_doctest + >>> test.test_doctest.test_doctest.sillySetup + True + +This test also has some (random) encoded (utf-8) unicode text: + + ÉÊÈà+ +This doesn't cause a problem in the tect surrounding the examples, but +we include it here (in this test text file) to make sure. :) diff --git a/Lib/test/test_doctest/test_doctest3.txt b/Lib/test/test_doctest/test_doctest3.txt new file mode 100644 index 00000000000..dd8557e57a5 --- /dev/null +++ b/Lib/test/test_doctest/test_doctest3.txt @@ -0,0 +1,5 @@ + +Here we check that `__file__` is provided: + + >>> type(__file__) + diff --git a/Lib/test/test_doctest/test_doctest4.txt b/Lib/test/test_doctest/test_doctest4.txt new file mode 100644 index 00000000000..0428e6f9632 --- /dev/null +++ b/Lib/test/test_doctest/test_doctest4.txt @@ -0,0 +1,11 @@ +This is a sample doctest in a text file that contains non-ASCII characters. +This file is encoded using UTF-8. + +In order to get this test to pass, we have to manually specify the +encoding. + + >>> 'föö' + 'f\xf6\xf6' + + >>> 'bÄ…r' + 'b\u0105r' diff --git a/Lib/test/test_doctest/test_doctest_errors.txt b/Lib/test/test_doctest/test_doctest_errors.txt new file mode 100644 index 00000000000..93c3c106e60 --- /dev/null +++ b/Lib/test/test_doctest/test_doctest_errors.txt @@ -0,0 +1,14 @@ +This is a sample doctest in a text file, in which all examples fail +or raise an exception. + + >>> 2 + 2 + 5 + >>> 1/0 + 1 + >>> def f(): + ... 2 + '2' + ... + >>> f() + 1 + >>> 2+*3 + 5 diff --git a/Lib/test/test_doctest/test_doctest_skip.txt b/Lib/test/test_doctest/test_doctest_skip.txt new file mode 100644 index 00000000000..06c23d06e60 --- /dev/null +++ b/Lib/test/test_doctest/test_doctest_skip.txt @@ -0,0 +1,6 @@ +This is a sample doctest in a text file, in which all examples are skipped. + + >>> 2 + 2 # doctest: +SKIP + 5 + >>> 2 + 2 # doctest: +SKIP + 4 diff --git a/Lib/test/test_doctest/test_doctest_skip2.txt b/Lib/test/test_doctest/test_doctest_skip2.txt new file mode 100644 index 00000000000..85e4938c346 --- /dev/null +++ b/Lib/test/test_doctest/test_doctest_skip2.txt @@ -0,0 +1,6 @@ +This is a sample doctest in a text file, in which some examples are skipped. + + >>> 2 + 2 # doctest: +SKIP + 5 + >>> 2 + 2 + 4 diff --git a/crates/vm/src/builtins/int.rs b/crates/vm/src/builtins/int.rs index 182333cea51..9b74b66e3a7 100644 --- a/crates/vm/src/builtins/int.rs +++ b/crates/vm/src/builtins/int.rs @@ -129,7 +129,7 @@ fn inner_mod(int1: &BigInt, int2: &BigInt, vm: &VirtualMachine) -> PyResult { fn inner_floordiv(int1: &BigInt, int2: &BigInt, vm: &VirtualMachine) -> PyResult { if int2.is_zero() { - Err(vm.new_zero_division_error("integer division by zero")) + Err(vm.new_zero_division_error("integer division or modulo by zero")) } else { Ok(vm.ctx.new_int(int1.div_floor(int2)).into()) } From 580f6e3f34659a89e4b26d076ba3a90199be0e2b Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 31 Dec 2025 09:47:59 +0900 Subject: [PATCH 0014/1575] Bump cranelift-* from 0.126.1 to 0.127.0 (#6571) Bumps [cranelift](https://github.com/bytecodealliance/wasmtime) from 0.126.1 to 0.127.0. - [Release notes](https://github.com/bytecodealliance/wasmtime/releases) - [Changelog](https://github.com/bytecodealliance/wasmtime/blob/main/RELEASES.md) - [Commits](https://github.com/bytecodealliance/wasmtime/commits) -------- Co-authored-by: copilot-swe-agent[bot] <198982749+Copilot@users.noreply.github.com> Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Co-authored-by: Copilot <198982749+Copilot@users.noreply.github.com> Co-authored-by: youknowone <69878+youknowone@users.noreply.github.com> --- Cargo.lock | 76 +++++++++++++++++++++---------------------- crates/jit/Cargo.toml | 6 ++-- 2 files changed, 41 insertions(+), 41 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 46c4dafe13b..89cace13b26 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -691,9 +691,9 @@ dependencies = [ [[package]] name = "cranelift" -version = "0.126.1" +version = "0.127.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "68971376deb1edf5e9c0ac77ef00479d740ce7a60e6181adb0648afe1dc7b8f4" +checksum = "513887fe45ce3979a4766ddc9992c3cdbf509add2a31906350649423a1d0d287" dependencies = [ "cranelift-codegen", "cranelift-frontend", @@ -702,42 +702,42 @@ dependencies = [ [[package]] name = "cranelift-assembler-x64" -version = "0.126.1" +version = "0.127.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "30054f4aef4d614d37f27d5b77e36e165f0b27a71563be348e7c9fcfac41eed8" +checksum = "8bd963a645179fa33834ba61fa63353998543b07f877e208da9eb47d4a70d1e7" dependencies = [ "cranelift-assembler-x64-meta", ] [[package]] name = "cranelift-assembler-x64-meta" -version = "0.126.1" +version = "0.127.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0beab56413879d4f515e08bcf118b1cb85f294129bb117057f573d37bfbb925a" +checksum = "3f6d5739c9dc6b5553ca758d78d87d127dd19f397f776efecf817b8ba8d0bb01" dependencies = [ "cranelift-srcgen", ] [[package]] name = "cranelift-bforest" -version = "0.126.1" +version = "0.127.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6d054747549a69b264d5299c8ca1b0dd45dc6bd0ee43f1edfcc42a8b12952c7a" +checksum = "ff402c11bb1c9652b67a3e885e84b1b8d00c13472c8fd85211e06a41a63c3e03" dependencies = [ "cranelift-entity", ] [[package]] name = "cranelift-bitset" -version = "0.126.1" +version = "0.127.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "98b92d481b77a7dc9d07c96e24a16f29e0c9c27d042828fdf7e49e54ee9819bf" +checksum = "769a0d88c2f5539e9c5536a93a7bf164b0dc68d91e3d00723e5b4ffc1440afdc" [[package]] name = "cranelift-codegen" -version = "0.126.1" +version = "0.127.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6eeccfc043d599b0ef1806942707fc51cdd1c3965c343956dc975a55d82a920f" +checksum = "d4351f721fb3b26add1c180f0a75c7474bab2f903c8b777c6ca65238ded59a78" dependencies = [ "bumpalo", "cranelift-assembler-x64", @@ -761,9 +761,9 @@ dependencies = [ [[package]] name = "cranelift-codegen-meta" -version = "0.126.1" +version = "0.127.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1174cdb9d9d43b2bdaa612a07ed82af13db9b95526bc2c286c2aec4689bcc038" +checksum = "61f86c0ba5b96713643f4dd0de0df12844de9c7bb137d6829b174b706939aa74" dependencies = [ "cranelift-assembler-x64-meta", "cranelift-codegen-shared", @@ -773,33 +773,33 @@ dependencies = [ [[package]] name = "cranelift-codegen-shared" -version = "0.126.1" +version = "0.127.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7d572be73fae802eb115f45e7e67a9ed16acb4ee683b67c4086768786545419a" +checksum = "f08605eee8d51fd976a970bd5b16c9529b51b624f8af68f80649ffb172eb85a4" [[package]] name = "cranelift-control" -version = "0.126.1" +version = "0.127.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e1587465cc84c5cc793b44add928771945f3132bbf6b3621ee9473c631a87156" +checksum = "623aab0a09e40f0cf0b5d35eb7832bae4c4f13e3768228e051a6c1a60e88ef5f" dependencies = [ "arbitrary", ] [[package]] name = "cranelift-entity" -version = "0.126.1" +version = "0.127.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "063b83448b1343e79282c3c7cbda7ed5f0816f0b763a4c15f7cecb0a17d87ea6" +checksum = "ea0f066e07e3bcbe38884cc5c94c32c7a90267d69df80f187d9dfe421adaa7c4" dependencies = [ "cranelift-bitset", ] [[package]] name = "cranelift-frontend" -version = "0.126.1" +version = "0.127.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "aa4461c2d2ca48bc72883f5f5c3129d9aefac832df1db824af9db8db3efee109" +checksum = "40865b02a0e52ca8e580ad64feef530cb1d05f6bb4972b4eef05e3eaeae81701" dependencies = [ "cranelift-codegen", "log", @@ -809,15 +809,15 @@ dependencies = [ [[package]] name = "cranelift-isle" -version = "0.126.1" +version = "0.127.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "acd811b25e18f14810d09c504e06098acc1d9dbfa24879bf0d6b6fb44415fc66" +checksum = "104b3c117ae513e9af1d90679842101193a5ccb96ac9f997966d85ea25be2852" [[package]] name = "cranelift-jit" -version = "0.126.1" +version = "0.127.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "01527663ba63c10509d7c87fd1f8495d21170ba35bf714f57271495689d8fde5" +checksum = "3aa5f855cfb8e4253ed2d0dfc1a0b6ebe4912e67aa8b7ee14026ff55ca17f1fe" dependencies = [ "anyhow", "cranelift-codegen", @@ -830,14 +830,14 @@ dependencies = [ "region", "target-lexicon", "wasmtime-internal-jit-icache-coherence", - "windows-sys 0.60.2", + "windows-sys 0.61.2", ] [[package]] name = "cranelift-module" -version = "0.126.1" +version = "0.127.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "72328edb49aeafb1655818c91c476623970cb7b8a89ffbdadd82ce7d13dedc1d" +checksum = "b1d01806b191b59f4fc4680293dd5f554caf2de5b62f95eff5beef7acb46c29c" dependencies = [ "anyhow", "cranelift-codegen", @@ -846,9 +846,9 @@ dependencies = [ [[package]] name = "cranelift-native" -version = "0.126.1" +version = "0.127.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2417046989d8d6367a55bbab2e406a9195d176f4779be4aa484d645887217d37" +checksum = "e5c54e0a358bc05b48f2032e1c320e7f468da068604f2869b77052eab68eb0fe" dependencies = [ "cranelift-codegen", "libc", @@ -857,9 +857,9 @@ dependencies = [ [[package]] name = "cranelift-srcgen" -version = "0.126.1" +version = "0.127.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8d039de901c8d928222b8128e1b9a9ab27b82a7445cb749a871c75d9cb25c57d" +checksum = "cc6f4b039f453b66c75e9f7886e5a2af96276e151f44dc19b24b58f9a0c98009" [[package]] name = "crc32fast" @@ -4293,21 +4293,21 @@ dependencies = [ [[package]] name = "wasmtime-internal-jit-icache-coherence" -version = "39.0.1" +version = "40.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b97ccd36e25390258ce6720add639ffe5a7d81a5c904350aa08f5bbc60433d22" +checksum = "0858b470463f3e7c73acd6049046049e64be17b98901c2db5047450cf83df1fe" dependencies = [ "anyhow", "cfg-if", "libc", - "windows-sys 0.60.2", + "windows-sys 0.61.2", ] [[package]] name = "wasmtime-internal-math" -version = "39.0.1" +version = "40.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cd1b856e1bbf0230ab560ba4204e944b141971adc4e6cdf3feb6979c1a7b7953" +checksum = "222e1a590ece4e898f20af1e541b61d2cb803f2557e7eaff23e6c1db5434454a" dependencies = [ "libm", ] diff --git a/crates/jit/Cargo.toml b/crates/jit/Cargo.toml index a79e48bac2d..fde10b7c006 100644 --- a/crates/jit/Cargo.toml +++ b/crates/jit/Cargo.toml @@ -17,9 +17,9 @@ num-traits = { workspace = true } thiserror = { workspace = true } libffi = { workspace = true } -cranelift = "0.126" -cranelift-jit = "0.126" -cranelift-module = "0.126" +cranelift = "0.127" +cranelift-jit = "0.127" +cranelift-module = "0.127" [dev-dependencies] rustpython-derive = { workspace = true } From 00ae4a175136541851f88b1c0108e565917bfbae Mon Sep 17 00:00:00 2001 From: "Jeong, YunWon" <69878+youknowone@users.noreply.github.com> Date: Wed, 31 Dec 2025 11:28:04 +0900 Subject: [PATCH 0015/1575] enhance error codec and exit code (#6602) --- crates/vm/src/vm/mod.rs | 23 +++++++++++++++++++---- 1 file changed, 19 insertions(+), 4 deletions(-) diff --git a/crates/vm/src/vm/mod.rs b/crates/vm/src/vm/mod.rs index ddbf7660f7d..b156e30b738 100644 --- a/crates/vm/src/vm/mod.rs +++ b/crates/vm/src/vm/mod.rs @@ -327,11 +327,17 @@ impl VirtualMachine { let line_buffering = buffered_stdio && (isatty || fd == 2); let newline = if cfg!(windows) { None } else { Some("\n") }; + // stderr uses backslashreplace error handler + let errors: Option<&str> = if fd == 2 { + Some("backslashreplace") + } else { + None + }; let stdio = self.call_method( &io, "TextIOWrapper", - (buf, (), (), newline, line_buffering, write_through), + (buf, (), errors, newline, line_buffering, write_through), )?; let mode = if write { "w" } else { "r" }; stdio.set_attr("mode", self.ctx.new_str(mode), self)?; @@ -853,7 +859,13 @@ impl VirtualMachine { [arg] => match_class!(match arg { ref i @ PyInt => { use num_traits::cast::ToPrimitive; - return i.as_bigint().to_u32().unwrap_or(0); + // Try u32 first, then i32 (for negative values), else -1 for overflow + let code = i + .as_bigint() + .to_u32() + .or_else(|| i.as_bigint().to_i32().map(|v| v as u32)) + .unwrap_or(-1i32 as u32); + return code; } arg => { if self.is_none(arg) { @@ -866,8 +878,11 @@ impl VirtualMachine { _ => args.as_object().repr(self).ok(), }; if let Some(msg) = msg { - let stderr = stdlib::sys::PyStderr(self); - writeln!(stderr, "{msg}"); + // Write using Python's write() to use stderr's error handler (backslashreplace) + if let Ok(stderr) = stdlib::sys::get_stderr(self) { + let _ = self.call_method(&stderr, "write", (msg,)); + let _ = self.call_method(&stderr, "write", ("\n",)); + } } 1 } else if exc.fast_isinstance(self.ctx.exceptions.keyboard_interrupt) { From 4cf5697e063361b0cb055199b10749576de92896 Mon Sep 17 00:00:00 2001 From: Ashwin Naren Date: Tue, 30 Dec 2025 18:57:42 -0800 Subject: [PATCH 0016/1575] Symboltable updates (#5861) * symboltable updates * Fix symboltable --------- Co-authored-by: Jeong YunWon --- crates/codegen/src/symboltable.rs | 92 +++++++++++++++++++++++++------ 1 file changed, 74 insertions(+), 18 deletions(-) diff --git a/crates/codegen/src/symboltable.rs b/crates/codegen/src/symboltable.rs index 1629e5fff38..07766373d5a 100644 --- a/crates/codegen/src/symboltable.rs +++ b/crates/codegen/src/symboltable.rs @@ -137,12 +137,12 @@ pub enum SymbolScope { bitflags! { #[derive(Copy, Clone, Debug, PartialEq)] pub struct SymbolFlags: u16 { - const REFERENCED = 0x001; - const ASSIGNED = 0x002; - const PARAMETER = 0x004; - const ANNOTATED = 0x008; - const IMPORTED = 0x010; - const NONLOCAL = 0x020; + const REFERENCED = 0x001; // USE + const ASSIGNED = 0x002; // DEF_LOCAL + const PARAMETER = 0x004; // DEF_PARAM + const ANNOTATED = 0x008; // DEF_ANNOT + const IMPORTED = 0x010; // DEF_IMPORT + const NONLOCAL = 0x020; // DEF_NONLOCAL // indicates if the symbol gets a value assigned by a named expression in a comprehension // this is required to correct the scope in the analysis. const ASSIGNED_IN_COMPREHENSION = 0x040; @@ -157,8 +157,12 @@ bitflags! { /// def method(self): /// return x // is_free_class /// ``` - const FREE_CLASS = 0x100; - const BOUND = Self::ASSIGNED.bits() | Self::PARAMETER.bits() | Self::IMPORTED.bits() | Self::ITER.bits(); + const FREE_CLASS = 0x100; // DEF_FREE_CLASS + const GLOBAL = 0x200; // DEF_GLOBAL + const COMP_ITER = 0x400; // DEF_COMP_ITER + const COMP_CELL = 0x800; // DEF_COMP_CELL + const TYPE_PARAM = 0x1000; // DEF_TYPE_PARAM + const BOUND = Self::ASSIGNED.bits() | Self::PARAMETER.bits() | Self::IMPORTED.bits() | Self::ITER.bits() | Self::TYPE_PARAM.bits(); } } @@ -611,6 +615,8 @@ struct SymbolTableBuilder { source_file: SourceFile, // Current scope's varnames being collected (temporary storage) current_varnames: Vec, + // Track if we're inside an iterable definition expression (for nested comprehensions) + in_iter_def_exp: bool, } /// Enum to indicate in what mode an expression @@ -634,6 +640,7 @@ impl SymbolTableBuilder { future_annotations: false, source_file, current_varnames: Vec::new(), + in_iter_def_exp: false, }; this.enter_scope("top", CompilerScope::Module, 0); this @@ -720,6 +727,23 @@ impl SymbolTableBuilder { } else { SymbolUsage::Parameter }; + + // Check for duplicate parameter names + let table = self.tables.last().unwrap(); + if table.symbols.contains_key(parameter.name.as_str()) { + return Err(SymbolTableError { + error: format!( + "duplicate parameter '{}' in function definition", + parameter.name + ), + location: Some( + self.source_file + .to_source_code() + .source_location(parameter.name.range.start(), PositionEncoding::Utf8), + ), + }); + } + self.register_ident(¶meter.name, usage) } @@ -871,6 +895,18 @@ impl SymbolTableBuilder { if let Some(alias) = &name.asname { // `import my_module as my_alias` self.register_ident(alias, SymbolUsage::Imported)?; + } else if name.name.as_str() == "*" { + // Star imports are only allowed at module level + if self.tables.last().unwrap().typ != CompilerScope::Module { + return Err(SymbolTableError { + error: "'import *' only allowed at module level".to_string(), + location: Some(self.source_file.to_source_code().source_location( + name.name.range.start(), + PositionEncoding::Utf8, + )), + }); + } + // Don't register star imports as symbols } else { // `import module` self.register_name( @@ -1162,7 +1198,12 @@ impl SymbolTableBuilder { range, .. }) => { + let was_in_iter_def_exp = self.in_iter_def_exp; + if context == ExpressionContext::IterDefinitionExp { + self.in_iter_def_exp = true; + } self.scan_comprehension("genexpr", elt, None, generators, *range)?; + self.in_iter_def_exp = was_in_iter_def_exp; } Expr::ListComp(ExprListComp { elt, @@ -1170,7 +1211,12 @@ impl SymbolTableBuilder { range, node_index: _, }) => { + let was_in_iter_def_exp = self.in_iter_def_exp; + if context == ExpressionContext::IterDefinitionExp { + self.in_iter_def_exp = true; + } self.scan_comprehension("genexpr", elt, None, generators, *range)?; + self.in_iter_def_exp = was_in_iter_def_exp; } Expr::SetComp(ExprSetComp { elt, @@ -1178,7 +1224,12 @@ impl SymbolTableBuilder { range, node_index: _, }) => { + let was_in_iter_def_exp = self.in_iter_def_exp; + if context == ExpressionContext::IterDefinitionExp { + self.in_iter_def_exp = true; + } self.scan_comprehension("genexpr", elt, None, generators, *range)?; + self.in_iter_def_exp = was_in_iter_def_exp; } Expr::DictComp(ExprDictComp { key, @@ -1187,7 +1238,12 @@ impl SymbolTableBuilder { range, node_index: _, }) => { + let was_in_iter_def_exp = self.in_iter_def_exp; + if context == ExpressionContext::IterDefinitionExp { + self.in_iter_def_exp = true; + } self.scan_comprehension("genexpr", key, Some(value), generators, *range)?; + self.in_iter_def_exp = was_in_iter_def_exp; } Expr::Call(ExprCall { func, @@ -1312,8 +1368,8 @@ impl SymbolTableBuilder { node_index: _, }) => { // named expressions are not allowed in the definition of - // comprehension iterator definitions - if let ExpressionContext::IterDefinitionExp = context { + // comprehension iterator definitions (including nested comprehensions) + if context == ExpressionContext::IterDefinitionExp || self.in_iter_def_exp { return Err(SymbolTableError { error: "assignment expression cannot be used in a comprehension iterable expression".to_string(), location: Some(self.source_file.to_source_code().source_location(target.range().start(), PositionEncoding::Utf8)), @@ -1745,6 +1801,7 @@ impl SymbolTableBuilder { } SymbolUsage::Global => { symbol.scope = SymbolScope::GlobalExplicit; + flags.insert(SymbolFlags::GLOBAL); } SymbolUsage::Used => { flags.insert(SymbolFlags::REFERENCED); @@ -1753,21 +1810,20 @@ impl SymbolTableBuilder { flags.insert(SymbolFlags::ITER); } SymbolUsage::TypeParam => { - // Type parameters are always cell variables in their scope - symbol.scope = SymbolScope::Cell; - flags.insert(SymbolFlags::ASSIGNED); + flags.insert(SymbolFlags::ASSIGNED | SymbolFlags::TYPE_PARAM); } } // and even more checking // it is not allowed to assign to iterator variables (by named expressions) - if flags.contains(SymbolFlags::ITER | SymbolFlags::ASSIGNED) - /*&& symbol.is_assign_named_expr_in_comprehension*/ + if flags.contains(SymbolFlags::ITER) + && flags.contains(SymbolFlags::ASSIGNED_IN_COMPREHENSION) { return Err(SymbolTableError { - error: - "assignment expression cannot be used in a comprehension iterable expression" - .to_string(), + error: format!( + "assignment expression cannot rebind comprehension iteration variable '{}'", + symbol.name + ), location, }); } From 37c47fca6ba9a411192059e5ef671a470b286fbf Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 31 Dec 2025 12:02:47 +0900 Subject: [PATCH 0017/1575] Bump qs and express in /wasm/demo (#6603) Bumps [qs](https://github.com/ljharb/qs) and [express](https://github.com/expressjs/express). These dependencies needed to be updated together. Updates `qs` from 6.13.0 to 6.14.1 - [Changelog](https://github.com/ljharb/qs/blob/main/CHANGELOG.md) - [Commits](https://github.com/ljharb/qs/compare/v6.13.0...v6.14.1) Updates `express` from 4.21.2 to 4.22.1 - [Release notes](https://github.com/expressjs/express/releases) - [Changelog](https://github.com/expressjs/express/blob/v4.22.1/History.md) - [Commits](https://github.com/expressjs/express/compare/4.21.2...v4.22.1) --- updated-dependencies: - dependency-name: qs dependency-version: 6.14.1 dependency-type: indirect - dependency-name: express dependency-version: 4.22.1 dependency-type: indirect ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- wasm/demo/package-lock.json | 48 ++++++++++++++++++++++++------------- 1 file changed, 32 insertions(+), 16 deletions(-) diff --git a/wasm/demo/package-lock.json b/wasm/demo/package-lock.json index 3acc105e3c3..eb6957b3c54 100644 --- a/wasm/demo/package-lock.json +++ b/wasm/demo/package-lock.json @@ -2205,40 +2205,40 @@ } }, "node_modules/express": { - "version": "4.21.2", - "resolved": "https://registry.npmjs.org/express/-/express-4.21.2.tgz", - "integrity": "sha512-28HqgMZAmih1Czt9ny7qr6ek2qddF4FclbMzwhCREB6OFfH+rXAnuNCwo1/wFvrtbgsQDb4kSbX9de9lFbrXnA==", + "version": "4.22.1", + "resolved": "https://registry.npmjs.org/express/-/express-4.22.1.tgz", + "integrity": "sha512-F2X8g9P1X7uCPZMA3MVf9wcTqlyNp7IhH5qPCI0izhaOIYXaW9L535tGA3qmjRzpH+bZczqq7hVKxTR4NWnu+g==", "dev": true, "license": "MIT", "dependencies": { "accepts": "~1.3.8", "array-flatten": "1.1.1", - "body-parser": "1.20.3", - "content-disposition": "0.5.4", + "body-parser": "~1.20.3", + "content-disposition": "~0.5.4", "content-type": "~1.0.4", - "cookie": "0.7.1", - "cookie-signature": "1.0.6", + "cookie": "~0.7.1", + "cookie-signature": "~1.0.6", "debug": "2.6.9", "depd": "2.0.0", "encodeurl": "~2.0.0", "escape-html": "~1.0.3", "etag": "~1.8.1", - "finalhandler": "1.3.1", - "fresh": "0.5.2", - "http-errors": "2.0.0", + "finalhandler": "~1.3.1", + "fresh": "~0.5.2", + "http-errors": "~2.0.0", "merge-descriptors": "1.0.3", "methods": "~1.1.2", - "on-finished": "2.4.1", + "on-finished": "~2.4.1", "parseurl": "~1.3.3", - "path-to-regexp": "0.1.12", + "path-to-regexp": "~0.1.12", "proxy-addr": "~2.0.7", - "qs": "6.13.0", + "qs": "~6.14.0", "range-parser": "~1.2.1", "safe-buffer": "5.2.1", - "send": "0.19.0", - "serve-static": "1.16.2", + "send": "~0.19.0", + "serve-static": "~1.16.2", "setprototypeof": "1.2.0", - "statuses": "2.0.1", + "statuses": "~2.0.1", "type-is": "~1.6.18", "utils-merge": "1.0.1", "vary": "~1.1.2" @@ -2271,6 +2271,22 @@ "dev": true, "license": "MIT" }, + "node_modules/express/node_modules/qs": { + "version": "6.14.1", + "resolved": "https://registry.npmjs.org/qs/-/qs-6.14.1.tgz", + "integrity": "sha512-4EK3+xJl8Ts67nLYNwqw/dsFVnCf+qR7RgXSK9jEEm9unao3njwMDdmsdvoKBKHzxd7tCYz5e5M+SnMjdtXGQQ==", + "dev": true, + "license": "BSD-3-Clause", + "dependencies": { + "side-channel": "^1.1.0" + }, + "engines": { + "node": ">=0.6" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, "node_modules/express/node_modules/range-parser": { "version": "1.2.1", "resolved": "https://registry.npmjs.org/range-parser/-/range-parser-1.2.1.tgz", From bd0aaf6f4f6aaf92283b3f83058ce20208759e67 Mon Sep 17 00:00:00 2001 From: Terry Tianlin Luan <44815845+terryluan12@users.noreply.github.com> Date: Wed, 31 Dec 2025 00:58:34 -0500 Subject: [PATCH 0018/1575] Updated the hmac + mailbox libraries + associated tests (#6608) * Updated hmac + test library * Updated mailbox library * Added mailbox library test (v3.13.10) --- Lib/hmac.py | 2 +- Lib/mailbox.py | 96 +- Lib/test/test_hmac.py | 8 + Lib/test/test_mailbox.py | 2495 ++++++++++++++++++++++++++++++++++++++ 4 files changed, 2588 insertions(+), 13 deletions(-) create mode 100644 Lib/test/test_mailbox.py diff --git a/Lib/hmac.py b/Lib/hmac.py index 8b4f920db95..8b4eb2fe741 100644 --- a/Lib/hmac.py +++ b/Lib/hmac.py @@ -53,7 +53,7 @@ def __init__(self, key, msg=None, digestmod=''): raise TypeError("key: expected bytes or bytearray, but got %r" % type(key).__name__) if not digestmod: - raise TypeError("Missing required parameter 'digestmod'.") + raise TypeError("Missing required argument 'digestmod'.") if _hashopenssl and isinstance(digestmod, (str, _functype)): try: diff --git a/Lib/mailbox.py b/Lib/mailbox.py index 70da07ed2e9..b00d9e8634c 100644 --- a/Lib/mailbox.py +++ b/Lib/mailbox.py @@ -395,6 +395,56 @@ def get_file(self, key): f = open(os.path.join(self._path, self._lookup(key)), 'rb') return _ProxyFile(f) + def get_info(self, key): + """Get the keyed message's "info" as a string.""" + subpath = self._lookup(key) + if self.colon in subpath: + return subpath.split(self.colon)[-1] + return '' + + def set_info(self, key, info: str): + """Set the keyed message's "info" string.""" + if not isinstance(info, str): + raise TypeError(f'info must be a string: {type(info)}') + old_subpath = self._lookup(key) + new_subpath = old_subpath.split(self.colon)[0] + if info: + new_subpath += self.colon + info + if new_subpath == old_subpath: + return + old_path = os.path.join(self._path, old_subpath) + new_path = os.path.join(self._path, new_subpath) + os.rename(old_path, new_path) + self._toc[key] = new_subpath + + def get_flags(self, key): + """Return as a string the standard flags that are set on the keyed message.""" + info = self.get_info(key) + if info.startswith('2,'): + return info[2:] + return '' + + def set_flags(self, key, flags: str): + """Set the given flags and unset all others on the keyed message.""" + if not isinstance(flags, str): + raise TypeError(f'flags must be a string: {type(flags)}') + # TODO: check if flags are valid standard flag characters? + self.set_info(key, '2,' + ''.join(sorted(set(flags)))) + + def add_flag(self, key, flag: str): + """Set the given flag(s) without changing others on the keyed message.""" + if not isinstance(flag, str): + raise TypeError(f'flag must be a string: {type(flag)}') + # TODO: check that flag is a valid standard flag character? + self.set_flags(key, ''.join(set(self.get_flags(key)) | set(flag))) + + def remove_flag(self, key, flag: str): + """Unset the given string flag(s) without changing others on the keyed message.""" + if not isinstance(flag, str): + raise TypeError(f'flag must be a string: {type(flag)}') + if self.get_flags(key): + self.set_flags(key, ''.join(set(self.get_flags(key)) - set(flag))) + def iterkeys(self): """Return an iterator over keys.""" self._refresh() @@ -540,6 +590,8 @@ def _refresh(self): for subdir in self._toc_mtimes: path = self._paths[subdir] for entry in os.listdir(path): + if entry.startswith('.'): + continue p = os.path.join(path, entry) if os.path.isdir(p): continue @@ -698,9 +750,13 @@ def flush(self): _sync_close(new_file) # self._file is about to get replaced, so no need to sync. self._file.close() - # Make sure the new file's mode is the same as the old file's - mode = os.stat(self._path).st_mode - os.chmod(new_file.name, mode) + # Make sure the new file's mode and owner are the same as the old file's + info = os.stat(self._path) + os.chmod(new_file.name, info.st_mode) + try: + os.chown(new_file.name, info.st_uid, info.st_gid) + except (AttributeError, OSError): + pass try: os.rename(new_file.name, self._path) except FileExistsError: @@ -778,10 +834,11 @@ def get_message(self, key): """Return a Message representation or raise a KeyError.""" start, stop = self._lookup(key) self._file.seek(start) - from_line = self._file.readline().replace(linesep, b'') + from_line = self._file.readline().replace(linesep, b'').decode('ascii') string = self._file.read(stop - self._file.tell()) msg = self._message_factory(string.replace(linesep, b'\n')) - msg.set_from(from_line[5:].decode('ascii')) + msg.set_unixfrom(from_line) + msg.set_from(from_line[5:]) return msg def get_string(self, key, from_=False): @@ -1089,10 +1146,24 @@ def __len__(self): """Return a count of messages in the mailbox.""" return len(list(self.iterkeys())) + def _open_mh_sequences_file(self, text): + mode = '' if text else 'b' + kwargs = {'encoding': 'ASCII'} if text else {} + path = os.path.join(self._path, '.mh_sequences') + while True: + try: + return open(path, 'r+' + mode, **kwargs) + except FileNotFoundError: + pass + try: + return open(path, 'x+' + mode, **kwargs) + except FileExistsError: + pass + def lock(self): """Lock the mailbox.""" if not self._locked: - self._file = open(os.path.join(self._path, '.mh_sequences'), 'rb+') + self._file = self._open_mh_sequences_file(text=False) _lock_file(self._file) self._locked = True @@ -1146,7 +1217,11 @@ def remove_folder(self, folder): def get_sequences(self): """Return a name-to-key-list dictionary to define each sequence.""" results = {} - with open(os.path.join(self._path, '.mh_sequences'), 'r', encoding='ASCII') as f: + try: + f = open(os.path.join(self._path, '.mh_sequences'), 'r', encoding='ASCII') + except FileNotFoundError: + return results + with f: all_keys = set(self.keys()) for line in f: try: @@ -1169,7 +1244,7 @@ def get_sequences(self): def set_sequences(self, sequences): """Set sequences using the given name-to-key-list dictionary.""" - f = open(os.path.join(self._path, '.mh_sequences'), 'r+', encoding='ASCII') + f = self._open_mh_sequences_file(text=True) try: os.close(os.open(f.name, os.O_WRONLY | os.O_TRUNC)) for name, keys in sequences.items(): @@ -1956,10 +2031,7 @@ def readlines(self, sizehint=None): def __iter__(self): """Iterate over lines.""" - while True: - line = self.readline() - if not line: - return + while line := self.readline(): yield line def tell(self): diff --git a/Lib/test/test_hmac.py b/Lib/test/test_hmac.py index 74ebcb2fe70..1726975e864 100644 --- a/Lib/test/test_hmac.py +++ b/Lib/test/test_hmac.py @@ -505,6 +505,14 @@ def test_exercise_all_methods(self): self.fail("Exception raised during normal usage of HMAC class.") +class UpdateTestCase(unittest.TestCase): + @hashlib_helper.requires_hashdigest('sha256') + def test_with_str_update(self): + with self.assertRaises(TypeError): + h = hmac.new(b"key", digestmod='sha256') + h.update("invalid update") + + class CopyTestCase(unittest.TestCase): @hashlib_helper.requires_hashdigest('sha256') diff --git a/Lib/test/test_mailbox.py b/Lib/test/test_mailbox.py new file mode 100644 index 00000000000..940baf39415 --- /dev/null +++ b/Lib/test/test_mailbox.py @@ -0,0 +1,2495 @@ +import os +import sys +import time +import stat +import socket +import email +import email.message +import re +import io +import tempfile +from test import support +from test.support import import_helper +from test.support import os_helper +from test.support import refleak_helper +from test.support import socket_helper +from test.support.testcase import ExtraAssertions +import unittest +import textwrap +import mailbox +import glob + + +if not socket_helper.has_gethostname: + raise unittest.SkipTest("test requires gethostname()") + + +class TestBase: + + all_mailbox_types = (mailbox.Message, mailbox.MaildirMessage, + mailbox.mboxMessage, mailbox.MHMessage, + mailbox.BabylMessage, mailbox.MMDFMessage) + + def _check_sample(self, msg): + # Inspect a mailbox.Message representation of the sample message + self.assertIsInstance(msg, email.message.Message) + self.assertIsInstance(msg, mailbox.Message) + for key, value in _sample_headers: + self.assertIn(value, msg.get_all(key)) + self.assertTrue(msg.is_multipart()) + self.assertEqual(len(msg.get_payload()), len(_sample_payloads)) + for i, payload in enumerate(_sample_payloads): + part = msg.get_payload(i) + self.assertIsInstance(part, email.message.Message) + self.assertNotIsInstance(part, mailbox.Message) + self.assertEqual(part.get_payload(), payload) + + def _delete_recursively(self, target): + # Delete a file or delete a directory recursively + if os.path.isdir(target): + os_helper.rmtree(target) + elif os.path.exists(target): + os_helper.unlink(target) + + +class TestMailbox(TestBase): + + maxDiff = None + + _factory = None # Overridden by subclasses to reuse tests + _template = 'From: foo\n\n%s\n' + + def setUp(self): + self._path = os_helper.TESTFN + self._delete_recursively(self._path) + self._box = self._factory(self._path) + + def tearDown(self): + self._box.close() + self._delete_recursively(self._path) + + def test_add(self): + # Add copies of a sample message + keys = [] + keys.append(self._box.add(self._template % 0)) + self.assertEqual(len(self._box), 1) + keys.append(self._box.add(mailbox.Message(_sample_message))) + self.assertEqual(len(self._box), 2) + keys.append(self._box.add(email.message_from_string(_sample_message))) + self.assertEqual(len(self._box), 3) + keys.append(self._box.add(io.BytesIO(_bytes_sample_message))) + self.assertEqual(len(self._box), 4) + keys.append(self._box.add(_sample_message)) + self.assertEqual(len(self._box), 5) + keys.append(self._box.add(_bytes_sample_message)) + self.assertEqual(len(self._box), 6) + with self.assertWarns(DeprecationWarning): + keys.append(self._box.add( + io.TextIOWrapper(io.BytesIO(_bytes_sample_message), encoding="utf-8"))) + self.assertEqual(len(self._box), 7) + self.assertEqual(self._box.get_string(keys[0]), self._template % 0) + for i in (1, 2, 3, 4, 5, 6): + self._check_sample(self._box[keys[i]]) + + _nonascii_msg = textwrap.dedent("""\ + From: foo + Subject: Falinaptár házhozszállítással. Már rendeltél? + + 0 + """) + + def test_add_invalid_8bit_bytes_header(self): + key = self._box.add(self._nonascii_msg.encode('latin-1')) + self.assertEqual(len(self._box), 1) + self.assertEqual(self._box.get_bytes(key), + self._nonascii_msg.encode('latin-1')) + + def test_invalid_nonascii_header_as_string(self): + subj = self._nonascii_msg.splitlines()[1] + key = self._box.add(subj.encode('latin-1')) + self.assertEqual(self._box.get_string(key), + 'Subject: =?unknown-8bit?b?RmFsaW5hcHThciBo4Xpob3pzeuFsbO104XNz' + 'YWwuIE3hciByZW5kZWx06Ww/?=\n\n') + + def test_add_nonascii_string_header_raises(self): + with self.assertRaisesRegex(ValueError, "ASCII-only"): + self._box.add(self._nonascii_msg) + self._box.flush() + self.assertEqual(len(self._box), 0) + self.assertMailboxEmpty() + + def test_add_that_raises_leaves_mailbox_empty(self): + class CustomError(Exception): ... + exc_msg = "a fake error" + + def raiser(*args, **kw): + raise CustomError(exc_msg) + support.patch(self, email.generator.BytesGenerator, 'flatten', raiser) + with self.assertRaisesRegex(CustomError, exc_msg): + self._box.add(email.message_from_string("From: Alphöso")) + self.assertEqual(len(self._box), 0) + self._box.close() + self.assertMailboxEmpty() + + _non_latin_bin_msg = textwrap.dedent("""\ + From: foo@bar.com + To: báz + Subject: Maintenant je vous présente mon collègue, le pouf célèbre + \tJean de Baddie + Mime-Version: 1.0 + Content-Type: text/plain; charset="utf-8" + Content-Transfer-Encoding: 8bit + + Да, они летÑÑ‚. + """).encode('utf-8') + + def test_add_8bit_body(self): + key = self._box.add(self._non_latin_bin_msg) + self.assertEqual(self._box.get_bytes(key), + self._non_latin_bin_msg) + with self._box.get_file(key) as f: + self.assertEqual(f.read(), + self._non_latin_bin_msg.replace(b'\n', + os.linesep.encode())) + self.assertEqual(self._box[key].get_payload(), + "Да, они летÑÑ‚.\n") + + def test_add_binary_file(self): + with tempfile.TemporaryFile('wb+') as f: + f.write(_bytes_sample_message) + f.seek(0) + key = self._box.add(f) + self.assertEqual(self._box.get_bytes(key).split(b'\n'), + _bytes_sample_message.split(b'\n')) + + def test_add_binary_nonascii_file(self): + with tempfile.TemporaryFile('wb+') as f: + f.write(self._non_latin_bin_msg) + f.seek(0) + key = self._box.add(f) + self.assertEqual(self._box.get_bytes(key).split(b'\n'), + self._non_latin_bin_msg.split(b'\n')) + + def test_add_text_file_warns(self): + with tempfile.TemporaryFile('w+', encoding='utf-8') as f: + f.write(_sample_message) + f.seek(0) + with self.assertWarns(DeprecationWarning): + key = self._box.add(f) + self.assertEqual(self._box.get_bytes(key).split(b'\n'), + _bytes_sample_message.split(b'\n')) + + def test_add_StringIO_warns(self): + with self.assertWarns(DeprecationWarning): + key = self._box.add(io.StringIO(self._template % "0")) + self.assertEqual(self._box.get_string(key), self._template % "0") + + def test_add_nonascii_StringIO_raises(self): + with self.assertWarns(DeprecationWarning): + with self.assertRaisesRegex(ValueError, "ASCII-only"): + self._box.add(io.StringIO(self._nonascii_msg)) + self.assertEqual(len(self._box), 0) + self._box.close() + self.assertMailboxEmpty() + + def test_remove(self): + # Remove messages using remove() + self._test_remove_or_delitem(self._box.remove) + + def test_delitem(self): + # Remove messages using __delitem__() + self._test_remove_or_delitem(self._box.__delitem__) + + def _test_remove_or_delitem(self, method): + # (Used by test_remove() and test_delitem().) + key0 = self._box.add(self._template % 0) + key1 = self._box.add(self._template % 1) + self.assertEqual(len(self._box), 2) + method(key0) + self.assertEqual(len(self._box), 1) + self.assertRaises(KeyError, lambda: self._box[key0]) + self.assertRaises(KeyError, lambda: method(key0)) + self.assertEqual(self._box.get_string(key1), self._template % 1) + key2 = self._box.add(self._template % 2) + self.assertEqual(len(self._box), 2) + method(key2) + self.assertEqual(len(self._box), 1) + self.assertRaises(KeyError, lambda: self._box[key2]) + self.assertRaises(KeyError, lambda: method(key2)) + self.assertEqual(self._box.get_string(key1), self._template % 1) + method(key1) + self.assertEqual(len(self._box), 0) + self.assertRaises(KeyError, lambda: self._box[key1]) + self.assertRaises(KeyError, lambda: method(key1)) + + def test_discard(self, repetitions=10): + # Discard messages + key0 = self._box.add(self._template % 0) + key1 = self._box.add(self._template % 1) + self.assertEqual(len(self._box), 2) + self._box.discard(key0) + self.assertEqual(len(self._box), 1) + self.assertRaises(KeyError, lambda: self._box[key0]) + self._box.discard(key0) + self.assertEqual(len(self._box), 1) + self.assertRaises(KeyError, lambda: self._box[key0]) + + def test_get(self): + # Retrieve messages using get() + key0 = self._box.add(self._template % 0) + msg = self._box.get(key0) + self.assertEqual(msg['from'], 'foo') + self.assertEqual(msg.get_payload(), '0\n') + self.assertIsNone(self._box.get('foo')) + self.assertIs(self._box.get('foo', False), False) + self._box.close() + self._box = self._factory(self._path) + key1 = self._box.add(self._template % 1) + msg = self._box.get(key1) + self.assertEqual(msg['from'], 'foo') + self.assertEqual(msg.get_payload(), '1\n') + + def test_getitem(self): + # Retrieve message using __getitem__() + key0 = self._box.add(self._template % 0) + msg = self._box[key0] + self.assertEqual(msg['from'], 'foo') + self.assertEqual(msg.get_payload(), '0\n') + self.assertRaises(KeyError, lambda: self._box['foo']) + self._box.discard(key0) + self.assertRaises(KeyError, lambda: self._box[key0]) + + def test_get_message(self): + # Get Message representations of messages + key0 = self._box.add(self._template % 0) + key1 = self._box.add(_sample_message) + msg0 = self._box.get_message(key0) + self.assertIsInstance(msg0, mailbox.Message) + self.assertEqual(msg0['from'], 'foo') + self.assertEqual(msg0.get_payload(), '0\n') + self._check_sample(self._box.get_message(key1)) + + def test_get_bytes(self): + # Get bytes representations of messages + key0 = self._box.add(self._template % 0) + key1 = self._box.add(_sample_message) + self.assertEqual(self._box.get_bytes(key0), + (self._template % 0).encode('ascii')) + self.assertEqual(self._box.get_bytes(key1), _bytes_sample_message) + + def test_get_string(self): + # Get string representations of messages + key0 = self._box.add(self._template % 0) + key1 = self._box.add(_sample_message) + self.assertEqual(self._box.get_string(key0), self._template % 0) + self.assertEqual(self._box.get_string(key1).split('\n'), + _sample_message.split('\n')) + + def test_get_file(self): + # Get file representations of messages + key0 = self._box.add(self._template % 0) + key1 = self._box.add(_sample_message) + with self._box.get_file(key0) as file: + data0 = file.read() + with self._box.get_file(key1) as file: + data1 = file.read() + self.assertEqual(data0.decode('ascii').replace(os.linesep, '\n'), + self._template % 0) + self.assertEqual(data1.decode('ascii').replace(os.linesep, '\n'), + _sample_message) + + def test_get_file_can_be_closed_twice(self): + # Issue 11700 + key = self._box.add(_sample_message) + f = self._box.get_file(key) + f.close() + f.close() + + def test_iterkeys(self): + # Get keys using iterkeys() + self._check_iteration(self._box.iterkeys, do_keys=True, do_values=False) + + def test_keys(self): + # Get keys using keys() + self._check_iteration(self._box.keys, do_keys=True, do_values=False) + + def test_itervalues(self): + # Get values using itervalues() + self._check_iteration(self._box.itervalues, do_keys=False, + do_values=True) + + def test_iter(self): + # Get values using __iter__() + self._check_iteration(self._box.__iter__, do_keys=False, + do_values=True) + + def test_values(self): + # Get values using values() + self._check_iteration(self._box.values, do_keys=False, do_values=True) + + def test_iteritems(self): + # Get keys and values using iteritems() + self._check_iteration(self._box.iteritems, do_keys=True, + do_values=True) + + def test_items(self): + # Get keys and values using items() + self._check_iteration(self._box.items, do_keys=True, do_values=True) + + def _check_iteration(self, method, do_keys, do_values, repetitions=10): + for value in method(): + self.fail("Not empty") + keys, values = [], [] + for i in range(repetitions): + keys.append(self._box.add(self._template % i)) + values.append(self._template % i) + if do_keys and not do_values: + returned_keys = list(method()) + elif do_values and not do_keys: + returned_values = list(method()) + else: + returned_keys, returned_values = [], [] + for key, value in method(): + returned_keys.append(key) + returned_values.append(value) + if do_keys: + self.assertEqual(len(keys), len(returned_keys)) + self.assertEqual(set(keys), set(returned_keys)) + if do_values: + count = 0 + for value in returned_values: + self.assertEqual(value['from'], 'foo') + self.assertLess(int(value.get_payload()), repetitions) + count += 1 + self.assertEqual(len(values), count) + + def test_contains(self): + # Check existence of keys using __contains__() + self.assertNotIn('foo', self._box) + key0 = self._box.add(self._template % 0) + self.assertIn(key0, self._box) + self.assertNotIn('foo', self._box) + key1 = self._box.add(self._template % 1) + self.assertIn(key1, self._box) + self.assertIn(key0, self._box) + self.assertNotIn('foo', self._box) + self._box.remove(key0) + self.assertNotIn(key0, self._box) + self.assertIn(key1, self._box) + self.assertNotIn('foo', self._box) + self._box.remove(key1) + self.assertNotIn(key1, self._box) + self.assertNotIn(key0, self._box) + self.assertNotIn('foo', self._box) + + def test_len(self, repetitions=10): + # Get message count + keys = [] + for i in range(repetitions): + self.assertEqual(len(self._box), i) + keys.append(self._box.add(self._template % i)) + self.assertEqual(len(self._box), i + 1) + for i in range(repetitions): + self.assertEqual(len(self._box), repetitions - i) + self._box.remove(keys[i]) + self.assertEqual(len(self._box), repetitions - i - 1) + + def test_set_item(self): + # Modify messages using __setitem__() + key0 = self._box.add(self._template % 'original 0') + self.assertEqual(self._box.get_string(key0), + self._template % 'original 0') + key1 = self._box.add(self._template % 'original 1') + self.assertEqual(self._box.get_string(key1), + self._template % 'original 1') + self._box[key0] = self._template % 'changed 0' + self.assertEqual(self._box.get_string(key0), + self._template % 'changed 0') + self._box[key1] = self._template % 'changed 1' + self.assertEqual(self._box.get_string(key1), + self._template % 'changed 1') + self._box[key0] = _sample_message + self._check_sample(self._box[key0]) + self._box[key1] = self._box[key0] + self._check_sample(self._box[key1]) + self._box[key0] = self._template % 'original 0' + self.assertEqual(self._box.get_string(key0), + self._template % 'original 0') + self._check_sample(self._box[key1]) + self.assertRaises(KeyError, + lambda: self._box.__setitem__('foo', 'bar')) + self.assertRaises(KeyError, lambda: self._box['foo']) + self.assertEqual(len(self._box), 2) + + def test_clear(self, iterations=10): + # Remove all messages using clear() + keys = [] + for i in range(iterations): + self._box.add(self._template % i) + for i, key in enumerate(keys): + self.assertEqual(self._box.get_string(key), self._template % i) + self._box.clear() + self.assertEqual(len(self._box), 0) + for i, key in enumerate(keys): + self.assertRaises(KeyError, lambda: self._box.get_string(key)) + + def test_pop(self): + # Get and remove a message using pop() + key0 = self._box.add(self._template % 0) + self.assertIn(key0, self._box) + key1 = self._box.add(self._template % 1) + self.assertIn(key1, self._box) + self.assertEqual(self._box.pop(key0).get_payload(), '0\n') + self.assertNotIn(key0, self._box) + self.assertIn(key1, self._box) + key2 = self._box.add(self._template % 2) + self.assertIn(key2, self._box) + self.assertEqual(self._box.pop(key2).get_payload(), '2\n') + self.assertNotIn(key2, self._box) + self.assertIn(key1, self._box) + self.assertEqual(self._box.pop(key1).get_payload(), '1\n') + self.assertNotIn(key1, self._box) + self.assertEqual(len(self._box), 0) + + def test_popitem(self, iterations=10): + # Get and remove an arbitrary (key, message) using popitem() + keys = [] + for i in range(10): + keys.append(self._box.add(self._template % i)) + seen = [] + for i in range(10): + key, msg = self._box.popitem() + self.assertIn(key, keys) + self.assertNotIn(key, seen) + seen.append(key) + self.assertEqual(int(msg.get_payload()), keys.index(key)) + self.assertEqual(len(self._box), 0) + for key in keys: + self.assertRaises(KeyError, lambda: self._box[key]) + + def test_update(self): + # Modify multiple messages using update() + key0 = self._box.add(self._template % 'original 0') + key1 = self._box.add(self._template % 'original 1') + key2 = self._box.add(self._template % 'original 2') + self._box.update({key0: self._template % 'changed 0', + key2: _sample_message}) + self.assertEqual(len(self._box), 3) + self.assertEqual(self._box.get_string(key0), + self._template % 'changed 0') + self.assertEqual(self._box.get_string(key1), + self._template % 'original 1') + self._check_sample(self._box[key2]) + self._box.update([(key2, self._template % 'changed 2'), + (key1, self._template % 'changed 1'), + (key0, self._template % 'original 0')]) + self.assertEqual(len(self._box), 3) + self.assertEqual(self._box.get_string(key0), + self._template % 'original 0') + self.assertEqual(self._box.get_string(key1), + self._template % 'changed 1') + self.assertEqual(self._box.get_string(key2), + self._template % 'changed 2') + self.assertRaises(KeyError, + lambda: self._box.update({'foo': 'bar', + key0: self._template % "changed 0"})) + self.assertEqual(len(self._box), 3) + self.assertEqual(self._box.get_string(key0), + self._template % "changed 0") + self.assertEqual(self._box.get_string(key1), + self._template % "changed 1") + self.assertEqual(self._box.get_string(key2), + self._template % "changed 2") + + def test_flush(self): + # Write changes to disk + self._test_flush_or_close(self._box.flush, True) + + def test_popitem_and_flush_twice(self): + # See #15036. + self._box.add(self._template % 0) + self._box.add(self._template % 1) + self._box.flush() + + self._box.popitem() + self._box.flush() + self._box.popitem() + self._box.flush() + + def test_lock_unlock(self): + # Lock and unlock the mailbox + self.assertFalse(os.path.exists(self._get_lock_path())) + self._box.lock() + self.assertTrue(os.path.exists(self._get_lock_path())) + self._box.unlock() + self.assertFalse(os.path.exists(self._get_lock_path())) + + def test_close(self): + # Close mailbox and flush changes to disk + self._test_flush_or_close(self._box.close, False) + + def _test_flush_or_close(self, method, should_call_close): + contents = [self._template % i for i in range(3)] + self._box.add(contents[0]) + self._box.add(contents[1]) + self._box.add(contents[2]) + oldbox = self._box + method() + if should_call_close: + self._box.close() + self._box = self._factory(self._path) + keys = self._box.keys() + self.assertEqual(len(keys), 3) + for key in keys: + self.assertIn(self._box.get_string(key), contents) + oldbox.close() + + def test_dump_message(self): + # Write message representations to disk + for input in (email.message_from_string(_sample_message), + _sample_message, io.BytesIO(_bytes_sample_message)): + output = io.BytesIO() + self._box._dump_message(input, output) + self.assertEqual(output.getvalue(), + _bytes_sample_message.replace(b'\n', os.linesep.encode())) + output = io.BytesIO() + self.assertRaises(TypeError, + lambda: self._box._dump_message(None, output)) + + def _get_lock_path(self): + # Return the path of the dot lock file. May be overridden. + return self._path + '.lock' + + +class TestMailboxSuperclass(TestBase, unittest.TestCase): + + def test_notimplemented(self): + # Test that all Mailbox methods raise NotImplementedException. + box = mailbox.Mailbox('path') + self.assertRaises(NotImplementedError, lambda: box.add('')) + self.assertRaises(NotImplementedError, lambda: box.remove('')) + self.assertRaises(NotImplementedError, lambda: box.__delitem__('')) + self.assertRaises(NotImplementedError, lambda: box.discard('')) + self.assertRaises(NotImplementedError, lambda: box.__setitem__('', '')) + self.assertRaises(NotImplementedError, lambda: box.iterkeys()) + self.assertRaises(NotImplementedError, lambda: box.keys()) + self.assertRaises(NotImplementedError, lambda: box.itervalues().__next__()) + self.assertRaises(NotImplementedError, lambda: box.__iter__().__next__()) + self.assertRaises(NotImplementedError, lambda: box.values()) + self.assertRaises(NotImplementedError, lambda: box.iteritems().__next__()) + self.assertRaises(NotImplementedError, lambda: box.items()) + self.assertRaises(NotImplementedError, lambda: box.get('')) + self.assertRaises(NotImplementedError, lambda: box.__getitem__('')) + self.assertRaises(NotImplementedError, lambda: box.get_message('')) + self.assertRaises(NotImplementedError, lambda: box.get_string('')) + self.assertRaises(NotImplementedError, lambda: box.get_bytes('')) + self.assertRaises(NotImplementedError, lambda: box.get_file('')) + self.assertRaises(NotImplementedError, lambda: '' in box) + self.assertRaises(NotImplementedError, lambda: box.__contains__('')) + self.assertRaises(NotImplementedError, lambda: box.__len__()) + self.assertRaises(NotImplementedError, lambda: box.clear()) + self.assertRaises(NotImplementedError, lambda: box.pop('')) + self.assertRaises(NotImplementedError, lambda: box.popitem()) + self.assertRaises(NotImplementedError, lambda: box.update((('', ''),))) + self.assertRaises(NotImplementedError, lambda: box.flush()) + self.assertRaises(NotImplementedError, lambda: box.lock()) + self.assertRaises(NotImplementedError, lambda: box.unlock()) + self.assertRaises(NotImplementedError, lambda: box.close()) + + +class TestMaildir(TestMailbox, unittest.TestCase): + + _factory = lambda self, path, factory=None: mailbox.Maildir(path, factory) + + def setUp(self): + TestMailbox.setUp(self) + if (os.name == 'nt') or (sys.platform == 'cygwin'): + self._box.colon = '!' + + def assertMailboxEmpty(self): + self.assertEqual(os.listdir(os.path.join(self._path, 'tmp')), []) + + def test_add_MM(self): + # Add a MaildirMessage instance + msg = mailbox.MaildirMessage(self._template % 0) + msg.set_subdir('cur') + msg.set_info('foo') + key = self._box.add(msg) + self.assertTrue(os.path.exists(os.path.join(self._path, 'cur', '%s%sfoo' % + (key, self._box.colon)))) + + def test_get_MM(self): + # Get a MaildirMessage instance + msg = mailbox.MaildirMessage(self._template % 0) + msg.set_subdir('cur') + msg.set_flags('RF') + key = self._box.add(msg) + msg_returned = self._box.get_message(key) + self.assertIsInstance(msg_returned, mailbox.MaildirMessage) + self.assertEqual(msg_returned.get_subdir(), 'cur') + self.assertEqual(msg_returned.get_flags(), 'FR') + + def test_set_MM(self): + # Set with a MaildirMessage instance + msg0 = mailbox.MaildirMessage(self._template % 0) + msg0.set_flags('TP') + key = self._box.add(msg0) + msg_returned = self._box.get_message(key) + self.assertEqual(msg_returned.get_subdir(), 'new') + self.assertEqual(msg_returned.get_flags(), 'PT') + msg1 = mailbox.MaildirMessage(self._template % 1) + self._box[key] = msg1 + msg_returned = self._box.get_message(key) + self.assertEqual(msg_returned.get_subdir(), 'new') + self.assertEqual(msg_returned.get_flags(), '') + self.assertEqual(msg_returned.get_payload(), '1\n') + msg2 = mailbox.MaildirMessage(self._template % 2) + msg2.set_info('2,S') + self._box[key] = msg2 + self._box[key] = self._template % 3 + msg_returned = self._box.get_message(key) + self.assertEqual(msg_returned.get_subdir(), 'new') + self.assertEqual(msg_returned.get_flags(), 'S') + self.assertEqual(msg_returned.get_payload(), '3\n') + + def test_consistent_factory(self): + # Add a message. + msg = mailbox.MaildirMessage(self._template % 0) + msg.set_subdir('cur') + msg.set_flags('RF') + key = self._box.add(msg) + + # Create new mailbox with + class FakeMessage(mailbox.MaildirMessage): + pass + box = mailbox.Maildir(self._path, factory=FakeMessage) + box.colon = self._box.colon + msg2 = box.get_message(key) + self.assertIsInstance(msg2, FakeMessage) + + def test_initialize_new(self): + # Initialize a non-existent mailbox + self.tearDown() + self._box = mailbox.Maildir(self._path) + self._check_basics() + self._delete_recursively(self._path) + self._box = self._factory(self._path, factory=None) + self._check_basics() + + def test_initialize_existing(self): + # Initialize an existing mailbox + self.tearDown() + for subdir in '', 'tmp', 'new', 'cur': + os.mkdir(os.path.normpath(os.path.join(self._path, subdir))) + self._box = mailbox.Maildir(self._path) + self._check_basics() + + def test_filename_leading_dot(self): + self.tearDown() + for subdir in '', 'tmp', 'new', 'cur': + os.mkdir(os.path.normpath(os.path.join(self._path, subdir))) + for subdir in 'tmp', 'new', 'cur': + fname = os.path.join(self._path, subdir, '.foo' + subdir) + with open(fname, 'wb') as f: + f.write(b"@") + self._box = mailbox.Maildir(self._path) + self.assertNotIn('.footmp', self._box) + self.assertNotIn('.foonew', self._box) + self.assertNotIn('.foocur', self._box) + self.assertEqual(list(self._box.iterkeys()), []) + + def _check_basics(self, factory=None): + # (Used by test_open_new() and test_open_existing().) + self.assertEqual(self._box._path, os.path.abspath(self._path)) + self.assertEqual(self._box._factory, factory) + for subdir in '', 'tmp', 'new', 'cur': + path = os.path.join(self._path, subdir) + self.assertTrue(os.path.isdir(path), f"Not a directory: {path!r}") + + def test_list_folders(self): + # List folders + self._box.add_folder('one') + self._box.add_folder('two') + self._box.add_folder('three') + self.assertEqual(len(self._box.list_folders()), 3) + self.assertEqual(set(self._box.list_folders()), + set(('one', 'two', 'three'))) + + def test_get_folder(self): + # Open folders + self._box.add_folder('foo.bar') + folder0 = self._box.get_folder('foo.bar') + folder0.add(self._template % 'bar') + self.assertTrue(os.path.isdir(os.path.join(self._path, '.foo.bar'))) + folder1 = self._box.get_folder('foo.bar') + self.assertEqual(folder1.get_string(folder1.keys()[0]), + self._template % 'bar') + + def test_add_and_remove_folders(self): + # Delete folders + self._box.add_folder('one') + self._box.add_folder('two') + self.assertEqual(len(self._box.list_folders()), 2) + self.assertEqual(set(self._box.list_folders()), set(('one', 'two'))) + self._box.remove_folder('one') + self.assertEqual(len(self._box.list_folders()), 1) + self.assertEqual(set(self._box.list_folders()), set(('two',))) + self._box.add_folder('three') + self.assertEqual(len(self._box.list_folders()), 2) + self.assertEqual(set(self._box.list_folders()), set(('two', 'three'))) + self._box.remove_folder('three') + self.assertEqual(len(self._box.list_folders()), 1) + self.assertEqual(set(self._box.list_folders()), set(('two',))) + self._box.remove_folder('two') + self.assertEqual(len(self._box.list_folders()), 0) + self.assertEqual(self._box.list_folders(), []) + + def test_clean(self): + # Remove old files from 'tmp' + foo_path = os.path.join(self._path, 'tmp', 'foo') + bar_path = os.path.join(self._path, 'tmp', 'bar') + with open(foo_path, 'w', encoding='utf-8') as f: + f.write("@") + with open(bar_path, 'w', encoding='utf-8') as f: + f.write("@") + self._box.clean() + self.assertTrue(os.path.exists(foo_path)) + self.assertTrue(os.path.exists(bar_path)) + foo_stat = os.stat(foo_path) + os.utime(foo_path, (time.time() - 129600 - 2, + foo_stat.st_mtime)) + self._box.clean() + self.assertFalse(os.path.exists(foo_path)) + self.assertTrue(os.path.exists(bar_path)) + + def test_create_tmp(self, repetitions=10): + # Create files in tmp directory + hostname = socket.gethostname() + if '/' in hostname: + hostname = hostname.replace('/', r'\057') + if ':' in hostname: + hostname = hostname.replace(':', r'\072') + pid = os.getpid() + pattern = re.compile(r"(?P