BUG: Datetime and Timestamp are not well supported by pd.eval #52559

semigodking · 2023-04-10T08:52:55Z

Pandas version checks

I have checked that this issue has not already been reported.
I have confirmed this bug exists on the latest version of pandas.
I have confirmed this bug exists on the main branch of pandas.

Reproducible Example

import pandas as pd
df = pd.DataFrame([[pd.to_datetime('now', utc=True)]], columns=['time1'])
pd.eval("pd.to_datetime('now', utc=True) - df['time1'] < pd.Timedelta(hours=1)", resolvers=[{'df': df, 'pd':pd}])
pd.eval("df['time1']-pd.Timedelta(hours=1)", resolvers=[{'df': df, 'pd':pd}])

Issue Description

When I try to calculate datetimes in by eval as code sample above, exceptions are raised. I know such calculations can be done without eval. However, for some reason, I would prefer this operation can be done via expression.

Exceptions:

--------------------------------------------------------------------------- TypeError Traceback (most recent call last) Cell In[25], line 1 ----> 1 pd.eval("pd.to_datetime('now', utc=True) - df['time1'] < pd.Timedelta(year=1)", resolvers=[{'df': df, 'pd':pd}])

File ~/venv/lib/python3.8/site-packages/pandas/core/computation/eval.py:336, in eval(expr, parser, engine, local_dict, global_dict, resolvers, level, target, inplace)
327 # get our (possibly passed-in) scope
328 env = ensure_scope(
329 level + 1,
330 global_dict=global_dict,
(...)
333 target=target,
334 )
--> 336 parsed_expr = Expr(expr, engine=engine, parser=parser, env=env)
338 if engine == "numexpr" and (
339 is_extension_array_dtype(parsed_expr.terms.return_type)
340 or getattr(parsed_expr.terms, "operand_types", None) is not None
(...)
344 )
345 ):
346 warnings.warn(
347 "Engine has switched to 'python' because numexpr does not support "
348 "extension array dtypes. Please set your engine to python manually.",
349 RuntimeWarning,
350 stacklevel=find_stack_level(),
351 )

File ~/venv/lib/python3.8/site-packages/pandas/core/computation/expr.py:809, in Expr.init(self, expr, engine, parser, env, level)
807 self.parser = parser
808 self._visitor = PARSERS[parser](self.env, self.engine, self.parser)
--> 809 self.terms = self.parse()

File ~/venv/lib/python3.8/site-packages/pandas/core/computation/expr.py:828, in Expr.parse(self)
824 def parse(self):
825 """
826 Parse an expression.
827 """
--> 828 return self._visitor.visit(self.expr)

File ~/venv/lib/python3.8/site-packages/pandas/core/computation/expr.py:415, in BaseExprVisitor.visit(self, node, **kwargs)
413 method = f"visit_{type(node).name}"
414 visitor = getattr(self, method)
--> 415 return visitor(node, **kwargs)

File ~/venv/lib/python3.8/site-packages/pandas/core/computation/expr.py:421, in BaseExprVisitor.visit_Module(self, node, **kwargs)
419 raise SyntaxError("only a single expression is allowed")
420 expr = node.body[0]
--> 421 return self.visit(expr, **kwargs)

File ~/venv/lib/python3.8/site-packages/pandas/core/computation/expr.py:415, in BaseExprVisitor.visit(self, node, **kwargs)
413 method = f"visit_{type(node).name}"
414 visitor = getattr(self, method)
--> 415 return visitor(node, **kwargs)

File ~/venv/lib/python3.8/site-packages/pandas/core/computation/expr.py:424, in BaseExprVisitor.visit_Expr(self, node, **kwargs)
423 def visit_Expr(self, node, **kwargs):
--> 424 return self.visit(node.value, **kwargs)

File ~/venv/lib/python3.8/site-packages/pandas/core/computation/expr.py:415, in BaseExprVisitor.visit(self, node, **kwargs)
413 method = f"visit_{type(node).name}"
414 visitor = getattr(self, method)
--> 415 return visitor(node, **kwargs)

File ~/venv/lib/python3.8/site-packages/pandas/core/computation/expr.py:719, in BaseExprVisitor.visit_Compare(self, node, **kwargs)
717 op = self.translate_In(ops[0])
718 binop = ast.BinOp(op=op, left=node.left, right=comps[0])
--> 719 return self.visit(binop)
721 # recursive case: we have a chained comparison, a CMP b CMP c, etc.
722 left = node.left

File ~/venv/lib/python3.8/site-packages/pandas/core/computation/expr.py:415, in BaseExprVisitor.visit(self, node, **kwargs)
413 method = f"visit_{type(node).name}"
414 visitor = getattr(self, method)
--> 415 return visitor(node, **kwargs)

File ~/venv/lib/python3.8/site-packages/pandas/core/computation/expr.py:535, in BaseExprVisitor.visit_BinOp(self, node, **kwargs)
534 def visit_BinOp(self, node, **kwargs):
--> 535 op, op_class, left, right = self._maybe_transform_eq_ne(node)
536 left, right = self._maybe_downcast_constants(left, right)
537 return self._maybe_evaluate_binop(op, op_class, left, right)

File ~/venv/lib/python3.8/site-packages/pandas/core/computation/expr.py:455, in BaseExprVisitor._maybe_transform_eq_ne(self, node, left, right)
453 def _maybe_transform_eq_ne(self, node, left=None, right=None):
454 if left is None:
--> 455 left = self.visit(node.left, side="left")
456 if right is None:
457 right = self.visit(node.right, side="right")

File ~/venv/lib/python3.8/site-packages/pandas/core/computation/expr.py:415, in BaseExprVisitor.visit(self, node, **kwargs)
413 method = f"visit_{type(node).name}"
414 visitor = getattr(self, method)
--> 415 return visitor(node, **kwargs)

File ~/venv/lib/python3.8/site-packages/pandas/core/computation/expr.py:537, in BaseExprVisitor.visit_BinOp(self, node, **kwargs)
535 op, op_class, left, right = self._maybe_transform_eq_ne(node)
536 left, right = self._maybe_downcast_constants(left, right)
--> 537 return self._maybe_evaluate_binop(op, op_class, left, right)

File ~/venv/lib/python3.8/site-packages/pandas/core/computation/expr.py:507, in BaseExprVisitor._maybe_evaluate_binop(self, op, op_class, lhs, rhs, eval_in_python, maybe_eval_in_python)
504 res = op(lhs, rhs)
506 if res.has_invalid_return_type:
--> 507 raise TypeError(
508 f"unsupported operand type(s) for {res.op}: "
509 f"'{lhs.type}' and '{rhs.type}'"
510 )
512 if self.engine != "pytables" and (
513 res.op in CMP_OPS_SYMS
514 and getattr(lhs, "is_datetime", False)
(...)
517 # all date ops must be done in python bc numexpr doesn't work
518 # well with NaT
519 return self._maybe_eval(res, self.binary_ops)

TypeError: unsupported operand type(s) for -: '<class 'pandas._libs.tslibs.timestamps.Timestamp'>' and 'datetime64[ns]'

pd.eval("df['time1']-pd.Timedelta(hours=1)", resolvers=[{'df': df, 'pd':pd}])
pd.eval("df['time1']-pd.Timedelta(hours=1)", resolvers=[{'df': df, 'pd':pd}])

--------------------------------------------------------------------------- TypeError Traceback (most recent call last) Cell In[27], line 1 ----> 1 pd.eval("df['time1']-pd.Timedelta(hours=1)", resolvers=[{'df': df, 'pd':pd}])