API/BUG: a timeseries/index localized to UTC when inserted into a Series/DataFrame will preserve the UTC timezone (rather than being a naive datetime64[ns]) as object dtype (GH8411)

Author: jreback

doc/source/basics.rst

@@ -1130,6 +1130,12 @@ You can easily produces tz aware transformations:
    stz
    stz.dt.tz
 
+You can also chain these types of operations:
+
+.. ipython:: python
+
+   s.dt.tz_localize('UTC').dt.tz_convert('US/Eastern')
+
 The ``.dt`` accessor works for period and timedelta dtypes.
 
 .. ipython:: python

doc/source/v0.15.0.txt

@@ -26,7 +26,11 @@ users upgrade to this version.
 
 - :ref:`Other Enhancements <whatsnew_0150.enhancements>`
 
-- :ref:`API Changes <whatsnew_0150.api>`, and :ref:`Rolling/Expanding Moments API Changes <whatsnew_0150.roll>`
+- :ref:`API Changes <whatsnew_0150.api>`
+
+- :ref:`Timezone API Change <whatsnew_0150.tz>`
+
+- :ref:`Rolling/Expanding Moments API Changes <whatsnew_0150.roll>`
 
 - :ref:`Performance Improvements <whatsnew_0150.performance>`
 
@@ -137,27 +141,6 @@ API changes
      In [3]: idx.isin(['a', 'c', 'e'], level=1)
      Out[3]: array([ True, False,  True,  True, False,  True], dtype=bool)
 
-- ``tz_localize(None)`` for tz-aware ``Timestamp`` and ``DatetimeIndex`` now removes timezone holding local time,
-  previously results in ``Exception`` or ``TypeError`` (:issue:`7812`)
-
-  .. ipython:: python
-
-     ts = Timestamp('2014-08-01 09:00', tz='US/Eastern')
-     ts
-     ts.tz_localize(None)
-
-     didx = DatetimeIndex(start='2014-08-01 09:00', freq='H', periods=10, tz='US/Eastern')
-     didx
-     didx.tz_localize(None)
-
-- ``tz_localize`` now accepts the ``ambiguous`` keyword which allows for passing an array of bools
-  indicating whether the date belongs in DST or not, 'NaT' for setting transition times to NaT,
-  'infer' for inferring DST/non-DST, and 'raise' (default) for an AmbiguousTimeError to be raised. See :ref:`the docs<timeseries.timezone_ambiguous>` for more details (:issue:`7943`)
-
-- ``DataFrame.tz_localize`` and ``DataFrame.tz_convert`` now accepts an optional ``level`` argument
-  for localizing a specific level of a MultiIndex (:issue:`7846`)
-- ``Timestamp.tz_localize`` and ``Timestamp.tz_convert`` now raise ``TypeError`` in error cases, rather than ``Exception`` (:issue:`8025`)
-- ``Timestamp.__repr__`` displays ``dateutil.tz.tzoffset`` info (:issue:`7907`)
 - ``merge``, ``DataFrame.merge``, and ``ordered_merge`` now return the same type
   as the ``left`` argument.  (:issue:`7737`)
 
@@ -305,6 +288,12 @@ You can easily produce tz aware transformations:
    stz
    stz.dt.tz
 
+You can also chain these types of operations:
+
+.. ipython:: python
+
+   s.dt.tz_localize('UTC').dt.tz_convert('US/Eastern')
+
 The ``.dt`` accessor works for period and timedelta dtypes.
 
 .. ipython:: python
@@ -324,6 +313,37 @@ The ``.dt`` accessor works for period and timedelta dtypes.
    s.dt.seconds
    s.dt.components
 
+.. _whatsnew_0150.tz:
+
+Timezone API changes
+~~~~~~~~~~~~~~~~~~~~
+
+- ``tz_localize(None)`` for tz-aware ``Timestamp`` and ``DatetimeIndex`` now removes timezone holding local time,
+  previously this resulted in ``Exception`` or ``TypeError`` (:issue:`7812`)
+
+  .. ipython:: python
+
+     ts = Timestamp('2014-08-01 09:00', tz='US/Eastern')
+     ts
+     ts.tz_localize(None)
+
+     didx = DatetimeIndex(start='2014-08-01 09:00', freq='H', periods=10, tz='US/Eastern')
+     didx
+     didx.tz_localize(None)
+
+- ``tz_localize`` now accepts the ``ambiguous`` keyword which allows for passing an array of bools
+  indicating whether the date belongs in DST or not, 'NaT' for setting transition times to NaT,
+  'infer' for inferring DST/non-DST, and 'raise' (default) for an AmbiguousTimeError to be raised. See :ref:`the docs<timeseries.timezone_ambiguous>` for more details (:issue:`7943`)
+
+- ``DataFrame.tz_localize`` and ``DataFrame.tz_convert`` now accepts an optional ``level`` argument
+  for localizing a specific level of a MultiIndex (:issue:`7846`)
+
+- ``Timestamp.tz_localize`` and ``Timestamp.tz_convert`` now raise ``TypeError`` in error cases, rather than ``Exception`` (:issue:`8025`)
+
+- a  timeseries/index localized to UTC when inserted into a Series/DataFrame will preserve the UTC timezone (rather than being a naive ``datetime64[ns]``) as ``object`` dtype (:issue:`8411`)
+
+- ``Timestamp.__repr__`` displays ``dateutil.tz.tzoffset`` info (:issue:`7907`)
+
 .. _whatsnew_0150.roll:
 
 Rolling/Expanding Moments API changes

pandas/core/frame.py

@@ -26,7 +26,7 @@
 from pandas.core.common import (isnull, notnull, PandasError, _try_sort,
                                 _default_index, _maybe_upcast, _is_sequence,
                                 _infer_dtype_from_scalar, _values_from_object,
-                                is_list_like, _get_dtype)
+                                is_list_like, _get_dtype, _maybe_box_datetimelike)
 from pandas.core.generic import NDFrame, _shared_docs
 from pandas.core.index import Index, MultiIndex, _ensure_index
 from pandas.core.indexing import (_maybe_droplevels,
@@ -1539,7 +1539,7 @@ def get_value(self, index, col, takeable=False):
 
         if takeable:
             series = self._iget_item_cache(col)
-            return series.values[index]
+            return _maybe_box_datetimelike(series.values[index])
 
         series = self._get_item_cache(col)
         engine = self.index._engine

pandas/tests/test_frame.py

@@ -3528,7 +3528,7 @@ def test_column_dups_indexing2(self):
 
         # GH 8363
         # datetime ops with a non-unique index
-        df = DataFrame({'A' : np.arange(5,dtype='int64'), 
+        df = DataFrame({'A' : np.arange(5,dtype='int64'),
                         'B' : np.arange(1,6,dtype='int64')},
                        index=[2,2,3,3,4])
         result = df.B-df.A
@@ -3654,6 +3654,18 @@ def test_constructor_with_datetimes(self):
         self.assertEqual(df.iat[0,0],dt)
         assert_series_equal(df.dtypes,Series({'End Date' : np.dtype('object') }))
 
+        # tz-aware (UTC and other tz's)
+        # GH 8411
+        dr = date_range('20130101',periods=3)
+        df = DataFrame({ 'value' : dr})
+        self.assertTrue(df.iat[0,0].tz is None)
+        dr = date_range('20130101',periods=3,tz='UTC')
+        df = DataFrame({ 'value' : dr})
+        self.assertTrue(str(df.iat[0,0].tz) == 'UTC')
+        dr = date_range('20130101',periods=3,tz='US/Eastern')
+        df = DataFrame({ 'value' : dr})
+        self.assertTrue(str(df.iat[0,0].tz) == 'US/Eastern')
+
         # GH 7822
         # preserver an index with a tz on dict construction
         i = date_range('1/1/2011', periods=5, freq='10s', tz = 'US/Eastern')
@@ -7654,7 +7666,7 @@ def test_fillna_dataframe(self):
                              index = list('VWXYZ'))
 
         assert_frame_equal(result, expected)
-        
+
     def test_fillna_columns(self):
         df = DataFrame(np.random.randn(10, 10))
         df.values[:, ::2] = np.nan
@@ -12791,8 +12803,8 @@ def test_consolidate_datetime64(self):
         df.starting = ser_starting.index
         df.ending = ser_ending.index
 
-        assert_array_equal(df.starting.values, ser_starting.index.values)
-        assert_array_equal(df.ending.values, ser_ending.index.values)
+        tm.assert_index_equal(pd.DatetimeIndex(df.starting), ser_starting.index)
+        tm.assert_index_equal(pd.DatetimeIndex(df.ending), ser_ending.index)
 
     def _check_bool_op(self, name, alternative, frame=None, has_skipna=True,
                        has_bool_only=False):

pandas/tests/test_series.py

@@ -135,6 +135,11 @@ def compare(s, name):
             freq_result = s.dt.freq
             self.assertEqual(freq_result, DatetimeIndex(s.values, freq='infer').freq)
 
+            # let's localize, then convert
+            result = s.dt.tz_localize('UTC').dt.tz_convert('US/Eastern')
+            expected = Series(DatetimeIndex(s.values).tz_localize('UTC').tz_convert('US/Eastern'),index=s.index)
+            tm.assert_series_equal(result, expected)
+
         # timedeltaindex
         for s in [Series(timedelta_range('1 day',periods=5)),
                   Series(timedelta_range('1 day 01:23:45',periods=5,freq='s')),
@@ -810,6 +815,15 @@ def test_constructor_dtype_datetime64(self):
         s = Series([pd.NaT, np.nan, '2013-08-05 15:30:00.000001'])
         self.assertEqual(s.dtype,'datetime64[ns]')
 
+        # tz-aware (UTC and other tz's)
+        # GH 8411
+        dr = date_range('20130101',periods=3)
+        self.assertTrue(Series(dr).iloc[0].tz is None)
+        dr = date_range('20130101',periods=3,tz='UTC')
+        self.assertTrue(str(Series(dr).iloc[0].tz) == 'UTC')
+        dr = date_range('20130101',periods=3,tz='US/Eastern')
+        self.assertTrue(str(Series(dr).iloc[0].tz) == 'US/Eastern')
+
     def test_constructor_periodindex(self):
         # GH7932
         # converting a PeriodIndex when put in a Series

pandas/tseries/index.py

@@ -744,7 +744,7 @@ def _to_embed(self, keep_tz=False):
 
         This is for internal compat
         """
-        if keep_tz and self.tz is not None and str(self.tz) != 'UTC':
+        if keep_tz and self.tz is not None:
             return self.asobject.values
         return self.values