Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Rewrite parser as part of new regex-syntax crate. #87

Merged
merged 4 commits into from
May 27, 2015
Merged

Rewrite parser as part of new regex-syntax crate. #87

Show file tree
Hide file tree
Changes from 2 commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
a66df89
Rewrite parser as part of new regex-syntax crate.
BurntSushi May 25, 2015
69e242d
Test regex-syntax crate in Travis.
BurntSushi May 25, 2015
6d5e909
Fixes from code review.
BurntSushi May 27, 2015
068514f
s/regex_syntax/regex-syntax
BurntSushi May 27, 2015
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Jump to
Jump to file
Failed to load files.
Diff view
Diff view
3 changes: 3 additions & 0 deletions .gitignore
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2,4 +2,7 @@
/Cargo.lock
/regex_macros/target
/regex_macros/Cargo.lock
/regex_syntax/target
/regex_syntax/Cargo.lock
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

It's probably also fine to just ignore everything called target and Cargo.lock here

/bench-log
.*.swp
2 changes: 2 additions & 0 deletions .travis.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -15,6 +15,8 @@ script:
cargo bench --manifest-path=regex_macros/Cargo.toml
)
- cargo doc
- cargo test --verbose --manifest-path=regex_syntax/Cargo.toml
- cargo doc --verbose --manifest-path=regex_syntax/Cargo.toml
after_success: |
[ $TRAVIS_BRANCH = master ] &&
[ $TRAVIS_PULL_REQUEST = false ] &&
Expand Down
8 changes: 8 additions & 0 deletions Cargo.toml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -21,8 +21,16 @@ path = "regex_macros/benches/bench_dynamic.rs"
test = false
bench = true

[dependencies.regex-syntax]
path = "regex_syntax"
version = "*"
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I personally prefer to write out this section like so nowadays:

[dependencies]
regex-syntax = { path = "regex_syntax", version = "*" }

Also, perhaps this could have a more strict version requirement than *? And finally, perhaps the folder could be called regex-syntax to match the crate name?

Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

And finally, perhaps the folder could be called regex-syntax to match the crate name?

Happy to oblige. I originally had that, but it looks very weird next to regex_macros, which I think is stuck with an underscore? :-(

👍 to everything else!

Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Yeah unfortunately we can't switch underscore-to-dash after the fact just yet :(

I would like to enable this kind of feature on crates.io though as I suspect it comes up more often than not!

Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

OK, hypens it is!


[dev-dependencies]
rand = "0.3"

[features]
pattern = []

[profile.bench]
opt-level = 3
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I think that 3 is the default, so perhaps this could just be specified to set lto = true

lto = true
138 changes: 67 additions & 71 deletions regex_macros/src/lib.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -36,10 +36,7 @@ use rustc::plugin::Registry;

use regex::Regex;
use regex::native::{
OneChar, CharClass, Any, Save, Jump, Split,
Match, EmptyBegin, EmptyEnd, EmptyWordBoundary,
Program, Dynamic, ExDynamic, Native,
FLAG_NOCASE, FLAG_MULTI, FLAG_DOTNL, FLAG_NEGATED,
Inst, Program, Dynamic, ExDynamic, Native,
simple_case_fold,
};

Expand Down Expand Up @@ -79,7 +76,9 @@ fn native(cx: &mut ExtCtxt, sp: codemap::Span, tts: &[ast::TokenTree])
// error is logged in 'parse' with cx.span_err
None => return DummyResult::any(sp),
};
let re = match Regex::new(&regex) {
// We use the largest possible size limit because this is happening at
// compile time. We trust the programmer.
let re = match Regex::with_size_limit(::std::usize::MAX, &regex) {
Ok(re) => re,
Err(err) => {
cx.span_err(sp, &err.to_string());
Expand Down Expand Up @@ -121,11 +120,10 @@ impl<'a> NfaGen<'a> {
None => cx.expr_none(self.sp),
}
);
let prefix_anchor =
match self.prog.insts[1] {
EmptyBegin(flags) if flags & FLAG_MULTI == 0 => true,
_ => false,
};
let prefix_anchor = match self.prog.insts[1] {
Inst::StartText => true,
_ => false,
};
let init_groups = self.vec_expr(0..num_cap_locs,
&mut |cx, _| cx.expr_none(self.sp));

Expand Down Expand Up @@ -338,49 +336,55 @@ fn exec<'t>(which: ::regex::native::MatchKind, input: &'t str,
let arms = self.prog.insts.iter().enumerate().map(|(pc, inst)| {
let nextpc = pc + 1;
let body = match *inst {
EmptyBegin(flags) => {
let cond =
if flags & FLAG_MULTI > 0 {
quote_expr!(self.cx,
self.chars.is_begin()
|| self.chars.prev == Some('\n')
)
} else {
quote_expr!(self.cx, self.chars.is_begin())
};
Inst::StartLine => {
quote_expr!(self.cx, {
nlist.add_empty($pc);
if $cond { self.add(nlist, $nextpc, &mut *groups) }
if self.chars.is_begin() || self.chars.prev == Some('\n') {
self.add(nlist, $nextpc, &mut *groups)
}
})
}
EmptyEnd(flags) => {
let cond =
if flags & FLAG_MULTI > 0 {
quote_expr!(self.cx,
self.chars.is_end()
|| self.chars.cur == Some('\n')
)
} else {
quote_expr!(self.cx, self.chars.is_end())
};
Inst::StartText => {
quote_expr!(self.cx, {
nlist.add_empty($pc);
if $cond { self.add(nlist, $nextpc, &mut *groups) }
if self.chars.is_begin() {
self.add(nlist, $nextpc, &mut *groups)
}
})
}
EmptyWordBoundary(flags) => {
let cond =
if flags & FLAG_NEGATED > 0 {
quote_expr!(self.cx, !self.chars.is_word_boundary())
} else {
quote_expr!(self.cx, self.chars.is_word_boundary())
};
Inst::EndLine => {
quote_expr!(self.cx, {
nlist.add_empty($pc);
if self.chars.is_end() || self.chars.cur == Some('\n') {
self.add(nlist, $nextpc, &mut *groups)
}
})
}
Inst::EndText => {
quote_expr!(self.cx, {
nlist.add_empty($pc);
if self.chars.is_end() {
self.add(nlist, $nextpc, &mut *groups)
}
})
}
Inst::WordBoundary => {
quote_expr!(self.cx, {
nlist.add_empty($pc);
if $cond { self.add(nlist, $nextpc, &mut *groups) }
if self.chars.is_word_boundary() {
self.add(nlist, $nextpc, &mut *groups)
}
})
}
Inst::NotWordBoundary => {
quote_expr!(self.cx, {
nlist.add_empty($pc);
if !self.chars.is_word_boundary() {
self.add(nlist, $nextpc, &mut *groups)
}
})
}
Save(slot) => {
Inst::Save(slot) => {
let save = quote_expr!(self.cx, {
let old = groups[$slot];
groups[$slot] = Some(self.ic);
Expand Down Expand Up @@ -411,20 +415,20 @@ fn exec<'t>(which: ::regex::native::MatchKind, input: &'t str,
})
}
}
Jump(to) => {
Inst::Jump(to) => {
quote_expr!(self.cx, {
nlist.add_empty($pc);
self.add(nlist, $to, &mut *groups);
})
}
Split(x, y) => {
Inst::Split(x, y) => {
quote_expr!(self.cx, {
nlist.add_empty($pc);
self.add(nlist, $x, &mut *groups);
self.add(nlist, $y, &mut *groups);
})
}
// For Match, OneChar, CharClass, Any
// For Match, OneChar, CharClass, Any, AnyNoNL
_ => quote_expr!(self.cx, nlist.add($pc, &*groups)),
};
self.arm_inst(pc, body)
Expand All @@ -439,7 +443,7 @@ fn exec<'t>(which: ::regex::native::MatchKind, input: &'t str,
let arms = self.prog.insts.iter().enumerate().map(|(pc, inst)| {
let nextpc = pc + 1;
let body = match *inst {
Match => {
Inst::Match => {
quote_expr!(self.cx, {
match self.which {
Exists => {
Expand All @@ -459,8 +463,8 @@ fn exec<'t>(which: ::regex::native::MatchKind, input: &'t str,
}
})
}
OneChar(c, flags) => {
if flags & FLAG_NOCASE > 0 {
Inst::OneChar { c, casei } => {
if casei {
let upc = simple_case_fold(c);
quote_expr!(self.cx, {
let upc = self.chars.prev.map(simple_case_fold);
Expand All @@ -476,45 +480,37 @@ fn exec<'t>(which: ::regex::native::MatchKind, input: &'t str,
})
}
}
CharClass(ref ranges, flags) => {
let negate = flags & FLAG_NEGATED > 0;
let casei = flags & FLAG_NOCASE > 0;
Inst::CharClass(ref cls) => {
let ranges: Vec<(char, char)> =
cls.iter().map(|r| (r.start, r.end)).collect();
let mranges = self.match_class(&ranges);
let get_char =
if casei {
if cls.is_case_insensitive() {
quote_expr!(
self.cx,
simple_case_fold(self.chars.prev.unwrap()))
} else {
quote_expr!(self.cx, self.chars.prev.unwrap())
};
let negcond =
if negate {
quote_expr!(self.cx, !found)
} else {
quote_expr!(self.cx, found)
};
let mranges = self.match_class(&ranges);
quote_expr!(self.cx, {
if self.chars.prev.is_some() {
let c = $get_char;
let found = $mranges;
if $negcond {
if $mranges {
self.add(nlist, $nextpc, caps);
}
}
})
}
Any(flags) => {
if flags & FLAG_DOTNL > 0 {
quote_expr!(self.cx, self.add(nlist, $nextpc, caps))
} else {
quote_expr!(self.cx, {
if self.chars.prev != Some('\n') {
self.add(nlist, $nextpc, caps)
}
()
})
}
Inst::Any => {
quote_expr!(self.cx, self.add(nlist, $nextpc, caps))
}
Inst::AnyNoNL => {
quote_expr!(self.cx, {
if self.chars.prev != Some('\n') {
self.add(nlist, $nextpc, caps);
}
()
})
}
// EmptyBegin, EmptyEnd, EmptyWordBoundary, Save, Jump, Split
_ => self.empty_block(),
Expand Down
6 changes: 3 additions & 3 deletions regex_macros/tests/tests.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -203,6 +203,8 @@ replace!(rep_named, replace_all,
"w1 w2 w3 w4", "$last $first$space", "w2 w1 w4 w3");
replace!(rep_trim, replace_all, "^[ \t]+|[ \t]+$", " \t trim me\t \t",
"", "trim me");
replace!(rep_number_hypen, replace, r"(.)(.)", "ab", "$1-$2", "a-b");
replace!(rep_number_underscore, replace, r"(.)(.)", "ab", "$1_$2", "a_b");

macro_rules! noparse(
($name:ident, $re:expr) => (
Expand All @@ -219,7 +221,6 @@ macro_rules! noparse(

noparse!(fail_double_repeat, "a**");
noparse!(fail_no_repeat_arg, "*");
noparse!(fail_no_repeat_arg_begin, "^*");
noparse!(fail_incomplete_escape, "\\");
noparse!(fail_class_incomplete, "[A-");
noparse!(fail_class_not_closed, "[A");
Expand All @@ -235,8 +236,7 @@ noparse!(fail_bad_capture_name, "(?P<na-me>)");
noparse!(fail_bad_flag, "(?a)a");
noparse!(fail_empty_alt_before, "|a");
noparse!(fail_empty_alt_after, "a|");
noparse!(fail_counted_big_exact, "a{1001}");
noparse!(fail_counted_big_min, "a{1001,}");
noparse!(fail_too_big, "a{10000000}");
noparse!(fail_counted_no_close, "a{1001");
noparse!(fail_unfinished_cap, "(?");
noparse!(fail_unfinished_escape, "\\");
Expand Down
13 changes: 13 additions & 0 deletions regex_syntax/Cargo.toml
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,13 @@
[package]
name = "regex-syntax"
version = "0.1.0"
authors = ["The Rust Project Developers"]
license = "MIT/Apache-2.0"
repository = "https://github.com/rust-lang/regex"
documentation = "http://doc.rust-lang.org/regex"
homepage = "https://github.com/rust-lang/regex"
description = "A regular expression parser (RE2 only)."

[dev-dependencies]
quickcheck = "*"
rand = "*"