Skip to content

bpo-47000: Add locale.getencoding() #32068

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
methane merged 13 commits into from
Apr 9, 2022
Merged

bpo-47000: Add locale.getencoding() #32068

Show file tree
Hide file tree
Changes from 1 commit
Commits
Show all changes
13 commits
Select commit Hold shift + click to select a range
cd4bccc
bpo-47000: Add locale.get_encoding()
methane Mar 20, 2022
aaa8956
get_encoding() -> getencoding()
methane Mar 31, 2022
104206a
fix TextIOWrapper.__init__
methane Mar 31, 2022
8b50eb1
get_encoding -> getencoding
methane Mar 31, 2022
a0204c2
Use _Py_STR for "utf-8"
methane Apr 4, 2022
08413f1
Update the doc.
methane Apr 6, 2022
f61b622
Add news
methane Apr 6, 2022
1c60e5b
Apply proposed changes.
methane Apr 6, 2022
7397493
Fix doc error.
methane Apr 6, 2022
44730dd
Apply suggestions from code review
methane Apr 7, 2022
7720b10
UTF-8 -> utf-8
methane Apr 7, 2022
104008d
Fix test_utf8_mode
methane Apr 7, 2022
a8e50bd
Update based on suggestions
methane Apr 8, 2022
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Next Next commit
bpo-47000: Add locale.get_encoding()
  • Loading branch information
@methane
methane committed Apr 4, 2022
commit cd4bccc51bc8eeca5ff4bd8329770b76c09ce03e
20 changes: 10 additions & 10 deletions Lib/locale.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -28,7 +28,7 @@
"setlocale", "resetlocale", "localeconv", "strcoll", "strxfrm",
"str", "atof", "atoi", "format", "format_string", "currency",
"normalize", "LC_CTYPE", "LC_COLLATE", "LC_TIME", "LC_MONETARY",
"LC_NUMERIC", "LC_ALL", "CHAR_MAX"]
"LC_NUMERIC", "LC_ALL", "CHAR_MAX", "get_encoding"]

def _strcoll(a,b):
""" strcoll(string,string) -> int.
Expand Down Expand Up @@ -637,27 +637,27 @@ def resetlocale(category=LC_ALL):


try:
from _locale import _get_locale_encoding
from _locale import get_encoding
except ImportError:
def _get_locale_encoding():
def get_encoding():
if hasattr(sys, 'getandroidapilevel'):
# On Android langinfo.h and CODESET are missing, and UTF-8 is
# always used in mbstowcs() and wcstombs().
return 'UTF-8'
if sys.flags.utf8_mode:
return 'UTF-8'
encoding = getdefaultlocale()[1]
if encoding is None:
# LANG not set, default conservatively to ASCII
encoding = 'ascii'
# LANG not set, default to UTF-8
encoding = 'UTF-8'
Comment thread
methane marked this conversation as resolved.
Outdated
return encoding

try:
CODESET
except NameError:
def getpreferredencoding(do_setlocale=True):
"""Return the charset that the user is likely using."""
return _get_locale_encoding()
if sys.flags.utf8_mode:
return 'UTF-8'
return get_encoding()
else:
# On Unix, if CODESET is available, use that.
def getpreferredencoding(do_setlocale=True):
Expand All @@ -667,15 +667,15 @@ def getpreferredencoding(do_setlocale=True):
return 'UTF-8'

if not do_setlocale:
return _get_locale_encoding()
return get_encoding()

old_loc = setlocale(LC_CTYPE)
try:
try:
setlocale(LC_CTYPE, "")
except Error:
pass
return _get_locale_encoding()
return get_encoding()
finally:
setlocale(LC_CTYPE, old_loc)

Expand Down
8 changes: 4 additions & 4 deletions Modules/_localemodule.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -773,14 +773,14 @@ _locale_bind_textdomain_codeset_impl(PyObject *module, const char *domain,


/*[clinic input]
_locale._get_locale_encoding
_locale.get_encoding

Get the current locale encoding.
[clinic start generated code]*/

static PyObject *
_locale__get_locale_encoding_impl(PyObject *module)
/*[clinic end generated code: output=e8e2f6f6f184591a input=513d9961d2f45c76]*/
_locale_get_encoding_impl(PyObject *module)
/*[clinic end generated code: output=7811932060493a1b input=163753befd041d95]*/
{
return _Py_GetLocaleEncodingObject();
}
Expand Down Expand Up @@ -811,7 +811,7 @@ static struct PyMethodDef PyLocale_Methods[] = {
_LOCALE_BIND_TEXTDOMAIN_CODESET_METHODDEF
#endif
#endif
_LOCALE__GET_LOCALE_ENCODING_METHODDEF
_LOCALE_GET_ENCODING_METHODDEF
{NULL, NULL}
};

Expand Down
16 changes: 8 additions & 8 deletions Modules/clinic/_localemodule.c.h
View file
Open in desktop

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

7 changes: 3 additions & 4 deletions Python/fileutils.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -93,6 +93,9 @@ _Py_device_encoding(int fd)

return PyUnicode_FromFormat("cp%u", (unsigned int)cp);
#else
if (_PyRuntime.preconfig.utf8_mode) {
return PyUnicode_FromString("utf-8"); //TODO: Use _Py_STR
}
return _Py_GetLocaleEncodingObject();
#endif
}
Expand Down Expand Up @@ -890,10 +893,6 @@ _Py_GetLocaleEncoding(void)
// and UTF-8 is always used in mbstowcs() and wcstombs().
return _PyMem_RawWcsdup(L"UTF-8");
#else
const PyPreConfig *preconfig = &_PyRuntime.preconfig;
if (preconfig->utf8_mode) {
return _PyMem_RawWcsdup(L"UTF-8");
}

#ifdef MS_WINDOWS
wchar_t encoding[23];
Expand Down
8 changes: 7 additions & 1 deletion Python/initconfig.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1779,7 +1779,13 @@ static PyStatus
config_get_locale_encoding(PyConfig *config, const PyPreConfig *preconfig,
wchar_t **locale_encoding)
{
wchar_t *encoding = _Py_GetLocaleEncoding();
wchar_t *encoding;
if (preconfig->utf8_mode) {
encoding = _PyMem_RawWcsdup(L"UTF-8");
}
else {
encoding = _Py_GetLocaleEncoding();
}
if (encoding == NULL) {
return _PyStatus_NO_MEMORY();
}
Expand Down