Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

BUG: mode not sorting values for arrow backed strings #55621

Merged
merged 9 commits into from Oct 25, 2023
Merged
Show file tree
Hide file tree
Changes from 5 commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Jump to
Jump to file
Failed to load files.
Diff view
Diff view
1 change: 1 addition & 0 deletions doc/source/whatsnew/v2.1.2.rst
Expand Up @@ -31,6 +31,7 @@ Bug fixes
- Fixed bug in :meth:`Index.insert` raising when inserting ``None`` into :class:`Index` with ``dtype="string[pyarrow_numpy]"`` (:issue:`55365`)
- Fixed bug in :meth:`Series.all` and :meth:`Series.any` not treating missing values correctly for ``dtype="string[pyarrow_numpy]"`` (:issue:`55367`)
- Fixed bug in :meth:`Series.floordiv` for :class:`ArrowDtype` (:issue:`55561`)
- Fixed bug in :meth:`Series.mode` not sorting values for arrow backed string dtype (:issue:`55621`)
- Fixed bug in :meth:`Series.rank` for ``string[pyarrow_numpy]`` dtype (:issue:`55362`)
- Fixed bug in :meth:`Series.str.extractall` for :class:`ArrowDtype` dtype being converted to object (:issue:`53846`)
- Fixed bug where PDEP-6 warning about setting an item of an incompatible dtype was being shown when creating a new conditional column (:issue:`55025`)
Expand Down
6 changes: 6 additions & 0 deletions pandas/compat/pyarrow.py
Expand Up @@ -7,13 +7,19 @@
try:
import pyarrow as pa

pa_installed = True
_palv = Version(Version(pa.__version__).base_version)
pa_version_under10p1 = _palv < Version("10.0.1")
pa_version_under11p0 = _palv < Version("11.0.0")
pa_version_under12p0 = _palv < Version("12.0.0")
pa_version_under13p0 = _palv < Version("13.0.0")
pa_version_under14p0 = _palv < Version("14.0.0")
except ImportError:
pa_installed = False
pa_version_under7p0 = True
pa_version_under8p0 = True
pa_version_under9p0 = True
pa_version_under10p0 = True
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Why do we need these?

Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

got in while merging main, sorry

pa_version_under10p1 = True
pa_version_under11p0 = True
pa_version_under12p0 = True
Expand Down
1 change: 1 addition & 0 deletions pandas/core/arrays/arrow/array.py
Expand Up @@ -1919,6 +1919,7 @@ def _mode(self, dropna: bool = True) -> Self:
if pa.types.is_temporal(pa_type):
most_common = most_common.cast(pa_type)

most_common = most_common.take(pc.array_sort_indices(most_common))
return type(self)(most_common)

def _maybe_convert_setitem_value(self, value):
Expand Down
4 changes: 2 additions & 2 deletions pandas/tests/extension/test_arrow.py
Expand Up @@ -1333,7 +1333,7 @@ def test_quantile(data, interpolation, quantile, request):

@pytest.mark.parametrize(
"take_idx, exp_idx",
[[[0, 0, 2, 2, 4, 4], [0, 4]], [[0, 0, 0, 2, 4, 4], [0]]],
[[[0, 0, 2, 2, 4, 4], [4, 0]], [[0, 0, 0, 2, 4, 4], [0]]],
ids=["multi_mode", "single_mode"],
)
def test_mode_dropna_true(data_for_grouping, take_idx, exp_idx):
Expand All @@ -1351,7 +1351,7 @@ def test_mode_dropna_false_mode_na(data):
expected = pd.Series([None], dtype=data.dtype)
tm.assert_series_equal(result, expected)

expected = pd.Series([None, data[0]], dtype=data.dtype)
expected = pd.Series([data[0], None], dtype=data.dtype)
result = expected.mode(dropna=False)
tm.assert_series_equal(result, expected)

Expand Down
20 changes: 17 additions & 3 deletions pandas/tests/groupby/test_groupby.py
Expand Up @@ -5,6 +5,7 @@
import numpy as np
import pytest

from pandas.compat.pyarrow import pa_installed
from pandas.errors import (
PerformanceWarning,
SpecificationError,
Expand Down Expand Up @@ -2541,7 +2542,10 @@ def test_groupby_column_index_name_lost(func):
"infer_string",
[
False,
True,
pytest.param(
True,
marks=pytest.mark.skipif(not pa_installed, reason="arrow not installed"),
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I just realized we have a td.skip_if_no mark as well

Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

does this work inside of pytest.param as well?

Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

It should, yeah. It returns a skipif marker

Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Yep, thx

),
],
)
def test_groupby_duplicate_columns(infer_string):
Expand Down Expand Up @@ -2773,13 +2777,23 @@ def test_rolling_wrong_param_min_period():
test_df.groupby("name")["val"].rolling(window=2, min_period=1).sum()


def test_by_column_values_with_same_starting_value():
@pytest.mark.parametrize(
"dtype",
[
object,
pytest.param(
"string[pyarrow_numpy]",
marks=pytest.mark.skipif(not pa_installed, reason="arrow not installed"),
),
],
)
def test_by_column_values_with_same_starting_value(dtype):
# GH29635
df = DataFrame(
{
"Name": ["Thomas", "Thomas", "Thomas John"],
"Credit": [1200, 1300, 900],
"Mood": ["sad", "happy", "happy"],
"Mood": Series(["sad", "happy", "happy"], dtype=dtype),
}
)
aggregate_details = {"Mood": Series.mode, "Credit": "sum"}
Expand Down