Go Language Tour #71 – Web Crawler

Go is doing my head in. This took me a long long time to get out:

package main

import (  
    "fmt"
)

type Fetcher interface {  
    // Fetch returns the body of URL and
    // a slice of URLs found on that page.
    Fetch(url string) (body string, urls []string, err error)
}

// Crawl uses fetcher to recursively crawl
// pages starting with url, to a maximum of depth.
func Crawl(url string, depth int, fetcher Fetcher) {  
    type Url struct {
        url string
        depth int
    }
    type HistoryReq struct {
        url string
        resultChan chan bool
    }

    historian := func() chan HistoryReq {
        history := map[string]bool{ url: true }
        historyReqChan := make(chan HistoryReq)
        go func() {
            for request := range historyReqChan {
                _, visited := history[request.url]
                if !visited {
                    history[request.url] = true
                } 
                request.resultChan <- visited
            }
        }()
        return historyReqChan
    }

    produce := func(urlChannel chan Url, urlRequest Url, historyReqChan chan HistoryReq, doneCounterChan chan bool) {
        getDepth, urlString := urlRequest.depth, urlRequest.url
        if (getDepth <= 0) {
            doneCounterChan <- false
            return
        }

        body, urls, err := fetcher.Fetch(urlString)
        if err != nil {
            fmt.Println(err)
        } else {
            fmt.Printf("found: %s %qn", urlString, body)
            for _, u := range urls {
                historyResultChan := make(chan bool)
                historyReq := HistoryReq{u, historyResultChan}
                historyReqChan <- historyReq
                if !<- historyResultChan {
                    urlChannel <- Url { u, getDepth - 1 }
                }                                                   
          }
        }
        doneCounterChan <- false
    }
    startWork := func(done chan bool, historyReqChan chan HistoryReq) chan Url {
        counter := 0
        doneCounterChan := make(chan bool)
        urlChannel := make(chan Url)
        go func() {
            for {
                select {
                case task := <- urlChannel:
                    counter++
                    go produce(urlChannel, task, historyReqChan, doneCounterChan)
                case <- doneCounterChan:
                    counter--
                    if counter == 0 {
                        done <- true
                    }
                }
            }
        }()
        return urlChannel
    }
    done := make(chan bool)    
    historyReqChan := historian()
    startWork(done, historyReqChan) <- Url{url, depth}

    <- done
    return
}

func main() {  
    Crawl("http://golang.org/", 4, fetcher)
}

// fakeFetcher is Fetcher that returns canned results.
type fakeFetcher map[string]*fakeResult

type fakeResult struct {  
    body string
    urls []string
}

func (f *fakeFetcher) Fetch(url string) (string, []string, error) {  
    if res, ok := (*f)[url]; ok {
        return res.body, res.urls, nil
    }
    return "", nil, fmt.Errorf("not found: %s", url)
}

// fetcher is a populated fakeFetcher.
var fetcher = &fakeFetcher{  
    "http://golang.org/": &fakeResult{
        "The Go Programming Language",
        []string{
            "http://golang.org/pkg/",
            "http://golang.org/cmd/",
        },
    },
    "http://golang.org/pkg/": &fakeResult{
        "Packages",
        []string{
            "http://golang.org/",
            "http://golang.org/cmd/",
            "http://golang.org/pkg/fmt/",
            "http://golang.org/pkg/os/",
        },
    },
    "http://golang.org/pkg/fmt/": &fakeResult{
        "Package fmt",
        []string{
            "http://golang.org/",
            "http://golang.org/pkg/",
        },
    },
    "http://golang.org/pkg/os/": &fakeResult{
        "Package os",
        []string{
            "http://golang.org/",
            "http://golang.org/pkg/",
        },
    },
}

Go’s use of channels for synchronisation takes some getting used to :/

Here’s an earlier, less parallel version:

package main

import (  
    "fmt"
)

type Fetcher interface {  
    // Fetch returns the body of URL and
    // a slice of URLs found on that page.
    Fetch(url string) (body string, urls []string, err error)
}

// Crawl uses fetcher to recursively crawl
// pages starting with url, to a maximum of depth.
func Crawl(url string, depth int, fetcher Fetcher) {  
    type Url struct {
        url string
        depth int
    }
    type History struct {
        urls map[string]bool
        lock chan bool
    }

    var produce func(urlChannel chan Url, done chan bool, history *History)
    produce = func(urlChannel chan Url, done chan bool, history *History) { 
        urlToRetrieve := <- urlChannel
        urlString, getDepth := urlToRetrieve.url, urlToRetrieve.depth
        if getDepth <= 0 {
            return
        }

        body, urls, err := fetcher.Fetch(urlString)
        if err != nil {
            fmt.Println(err)
        } else {
            fmt.Printf("found: %s %qn", urlString, body)
            <- history.lock 
            for _, u := range urls {
                if _, visited := history.urls[u]; !visited { 
                    history.urls[u] = true
                    go produce(urlChannel, done, history)
                    urlChannel <- Url { u, getDepth - 1 }
                }                                                   
            }
            select {
            case history.lock <- true:
            default:                      
                done <- true;
            }
        }
    }

    history := &History{ map[string]bool{ url: true }, make(chan bool) }
    urlChannel := make(chan Url)
    done := make(chan bool)

    go produce(urlChannel, done, history)
    urlChannel <- Url{url, depth}
    history.lock <- true

    <- done
    return
}

func main() {  
    Crawl("http://golang.org/", 4, fetcher)
}

// fakeFetcher is Fetcher that returns canned results.
type fakeFetcher map[string]*fakeResult

type fakeResult struct {  
    body string
    urls []string
}

func (f *fakeFetcher) Fetch(url string) (string, []string, error) {  
    if res, ok := (*f)[url]; ok {
        return res.body, res.urls, nil
    }
    return "", nil, fmt.Errorf("not found: %s", url)
}

// fetcher is a populated fakeFetcher.
var fetcher = &fakeFetcher{  
    "http://golang.org/": &fakeResult{
        "The Go Programming Language",
        []string{
            "http://golang.org/pkg/",
            "http://golang.org/cmd/",
        },
    },
    "http://golang.org/pkg/": &fakeResult{
        "Packages",
        []string{
            "http://golang.org/",
            "http://golang.org/cmd/",
            "http://golang.org/pkg/fmt/",
            "http://golang.org/pkg/os/",
        },
    },
    "http://golang.org/pkg/fmt/": &fakeResult{
        "Package fmt",
        []string{
            "http://golang.org/",
            "http://golang.org/pkg/",
        },
    },
    "http://golang.org/pkg/os/": &fakeResult{
        "Package os",
        []string{
            "http://golang.org/",
            "http://golang.org/pkg/",
        },
    },
}

 

PAF #4 Geo Party – Location Based Rooms

Working from home can be a lonely experience. There isn’t anyone to have coffee with or bounce ideas off. Things you usually take for granted, like being able to catch up become alot harder when everyone else is working or in a different timezone. That’s not to say there aren’t any avenues for human interaction, going out and working in public areas is always an option.

Image courtesy of FreeDigitalPhotos.net


So close yet so far
Image courtesy of FreeDigitalPhotos.net

Even when working in public areas, you’re close to folks, but not quite either. Walking up to someone random and starting a conversation is a tricky proposition for most. Interactions with strangers is not a new idea though; mediums like IRC have been around for ages.

IRC chatrooms had some nifty features for their time. Though you had to connect to a server, you could initiate a direct chat or transfer using something called Direct-Client-to-Client. In fact, many clients supported scripts that allowed you to run bots like fileservers and trivia games.

When MSN Messenger came along, you could even do things like initiate games between clients (Solitaire showdown anyone?) which was still alot easier than what you’d probably have to do if you wanted to do a two player game on your phone today.

So here’s what I hope PAF #4 will look like.

  • Multiple folks jump online to a site, based on their location a chatroom is created or joined
  • They can chat with each other (multi party chat has been done to the death, which should help :P)
  • They can send files to each other
  • They can see in relative terms how far they are to each other (e.g. 25m away)
  • They can enter an email address or login with FB/G+ which will pull down their info/gravatar
  • They can join another chat room if they know its secret

Initially I thought of using Go, but it turns out there isn’t any free online hosting available for it. Too efficient resource utilisation maybe? So I might try to get a micro EC2 instance or switch to something else.

PAF #3 SvenskaOrd Show and Tell

You can grab a copy of the apk (~5mb) , or build it yourself from the repo . The repo’s front page also has some explanation about how to use the app.

Its appearance hasn’t changed much from the midway point, it was mostly adding new features and backporting compatibility. Yes, it is compatible with Gingerbread but at some cost.

SvenskaOrd was interesting for a number of reasons. I started off with the idea that I would be able to do the backend first, which would make TDD easier and hook it up to the frontend, where the dev cycle was slower. Coming from using Corona in BishiBashi , where changes were reflected in seconds, using the Android SDK’s native simulator felt like watching paint dry at times. The sequence of events over the fortnight went something like this:

  • Marty came up with some mockups
  • I sketched out a data model
  • In Android Studio, used TDD to build out the logic independent of Android itself. Persistence still in-memory, just enough to make tests pass [expand title=”…”]
    • Unfortunately, the new Gradle based build system doesn’t quite support the notion of unit tests. Only instrumentTests, where the tests are run in the simulator and have access to the app classes under test.
    • The fact that the simulator is involved is bad enough, but the notion of unit tests is not even supported, i.e. you have to jump through numerous hoops to get simulator-independent unit tests working (much of the build-hackery involved can now be simplified thanks to the gradle-android-test-plugin).
    • Even then, you can’t seem to run unit tests and get results natively within Android Studio, the best anyone seems to be able to do is to run a Gradle task and inspect the test report

    [/expand]

  • Started work on the UI, calling backend methods
  • Introduced ORMLite to do persistence  [expand title=”…”]

    • This is where things really started to get hairy. Testing database interactions usually requires the simulator as its associated classes require an application context. Trying to mock out the provided android framework jars in the SDK is no help because they are mostly stubs and the implementation is only available after being linked on the actual device or simulator. You know you’ve run into this when you hit ‘java.lang.RuntimeException: Stub!'.
    • Thankfully, there is an awesome project called Robolectric that provides minimal stubbed versions of core framework classes, even allowing you to shadow specific classes if you need to. This seems to be the canonical library to use for Android unit testing when framework classes are involved.

    [/expand]

  • Realise that much of Android’s provided classes for view-to-db adapters assume you’re going with the framework and using ContentProviders / SomeCursorAdapter  / CursorLoader for async database access. This is somewhat at odds with ORMs which return you object graphs. I ended up Implementing ContentProviders that return Cursors from ORMLite’s CloseableIterators. [expand title=”…”]

    • While OrmLite has support for turning its CloseableIterators to Android cursors for use in CursorAdapters, it doesn’t help cases where you need to traverse the object graph and all you have is a cursor. If you were handcrafting your own SQL, you would include all of the object graph you need in your projection so the cursor has access, but OrmLite doesn’t support projections across multiple tables; you can only select columns from a single table.
    • As an alternative, you can simply roll your own AsyncTaskLoader to retrieve object graph collections from OrmLite, but you lose the flexibility of iterable collections

    [/expand]

  • Then introduce Roboguice to try and remove much boilerplate view retrieval. And saddened to realise that it actually requires the use of Android’s support library [expand title=”…”]

    • The support library adds newer features in a way that is compatible with older Android versions. For the most part this involved changing the inheritance hierarchy and implemented interfaces from something like app.android.Activity to app.android.support.v4.Activity .
    • Initially I thought this was a good idea anyway, since the majority of devices were still on Gingerbread when I last looked a year or two ago. How things have changed – http://developer.android.com/about/dashboards/index.html . The majority of devices are now ICS (API 15) or newer. Still, it seemed like a good exercise so I went ahead

    [/expand]

  • Introduce ActionBarSherlock (ABS) for better backward compatibility as I was using the actionbar for navigation, then realise [expand title=”…”]

    • You can’t use local aars without some build-hackery
    • RoboGuice uses inheritance. ActionBarSherlock uses inheritance. Multiple inheritance is not cool. But someone’s already done the grunt work to combine the hierarchies in roboguice-sherlock
    • You might get compile and runtime goodness with older devices using the support libraries and ABS, but you cannot prevent themes from missing 🙁 It turns out that the default theme for ICS+ devices is Holo , but it’s not available on Gingerbread or Froyo.
    • The kicker is ABS used to bundle a backported version of the Holo theme so it was available, but removed the backported dialog theme in newer versions because it wasn’t relevant to the library itself. I agree with the motivation, and sure, it’s just duplicating some xml and assets, but it certainly would have been nice to offer it as a separate download since they had already done the legwork.

    [/expand]

  • Discover a bunch of bugs because I hadn’t really tested the UI bits. TDD kind of fell by the wayside when I started hooking up UI components.

I had thought it would be a straightforward affair, given the large number of developers already on Android, but there’s still enough there that could probably span a few blogs or series of howto entries. Fleshing out entries takes time. Concrete examples and walkthroughs take time. It all just takes time, but perhaps I’ll do it as a PAF some time in the not too distant future.

I’ll probably put it on the app store after Marty’s nutted out the bugs but do download and have a play with it too. Issue trackers are enabled on all the repositories I’ve made public 😛

Making Makeup Work and Civilised Eyebrows

I am going to talk about makeup stuff today. Yes, that girly talk, you guys should still read on though. I promise it won’t be just skin deep…

This post will not be complete without the before and after shots, so here they are, left without makeup, right with makeup:

beforeafter

I think it’s fair to say that many girls in IT are late-adopters in the makeup market, me included. The product types themselves baffled me for quite a long time – eyeshadow, foundation, powder, bb cream, eye curler, bronzer, eyebrow pencils, etcetera etcetera. Not to mention the brands supplying those products. What I tend to associate when I heard about these products was, money, money, money and money. And more often than not, it also conjured some feeling of disappointment caused by buying something that ended up useless, only to be thrown in the bin five minutes after. Like buying lipstick that I thought would make me look great (read: look older), only to find that the colour made me look like a prostitute instead. Bin.

Despite the frequent experimental failures, I know that somehow, I will have to make it work for me. Yep, make the makeup work (can’t resist!). Men don’t have to put on makeup, good for them, but women can’t really get away from it. I’m not trying to be sexist, because a study done in 2009 illustrated nicely why makeup is only for the female sex:

‘Given this sex difference in contrast, Russell found a connection between the application of cosmetics and how it consistently increases facial contrast. Female faces wearing cosmetics have greater facial contrast than the same faces not wearing cosmetics. Russell noted that female facial beauty has been closely linked to sex differences, with femininity considered attractive. His results suggest that cosmetics may function in part by exaggerating a sexually dimorphic attribute to make the face appear more feminine and attractive.

“Cosmetics are typically used in precisely the correct way to exaggerate this difference, ” Russell said. “Making the eyes and lips darker without changing the surrounding skin increases the facial contrast. Femininity and attractiveness are highly correlated, so making a face more feminine also makes it more attractive.”

091020153100-large
In the photo, “Illusion of Sex,” two faces are perceived as male and female. However, both faces are actually versions of the same androgynous face. One face was created by increasing the contrast of the androgynous face, while the other face was created by decreasing the contrast. The face with more contrast is perceived as female, while the face with less contrast is perceived as male. This demonstrates that contrast is an important cue for perceiving the sex of a face, with greater contrast appearing feminine, and lesser contrast appearing masculine.’

So if wearing makeup objectively increase attractiveness for women, then I suppose the reasons to apply makeup everyday will be very similar to the reasons why we have showers or why we wear something decent. The quote I hear a lot is “Eat to please thyself, but dress to please others” (Benjamin Franklin). If you want to look your best, spending time on makeup can be more effective than spending time and money on other items like dresses or shoes. This is because people will look at your face more often than your shoes (wanna bet?). Applying foundations and powder makes your face looks brighter, looks healthier and most importantly, not pale. Lipstick does a great job in making the entire face appear fresh, even if internally I’m kind of sick *cough*. When I don’t have enough sleep the night before, concealers will fix up the bags under my eyes, they’re gone in a puff!

I have one monolid, and one normal eyelid. This drove me nuts, because I couldn’t really follow eye makeup advice for monolids, nor could I follow the advice for normal eyelid. It took a lot of trial and error using sticky tape, eye sticker, and eye glue to “fix” the monolid to make it look somewhat like a normal one (guess which method works best?). One of my aunt apparently used to have monolids when she was younger, so she cut sticky tape in the shape of the normal eyelids she wanted, and persevered to keep them on her eyes for the entire day, every day, for over a year. She was a real perfectionist and methodical. She got the double eyelids at the end, it was epic hardwork.

Good makeup brings the best of your facial features. I have always thought of my eyebrows to be dull, no more than messy pile of crumpled hairs. But with a little bit of shaping, and some brow zings, voila they turned to be something else!

IMG_7017

I love my madeup civilised eyebrows now, they’re kinda cool 🙂

Of course there’s also the negative side of makeup. Once I’ve decided to use makeup in a work setting, I have to consistently apply them because I will look quite different without it. This get quite repetitive and boring when I do it for 15 minutes each day, five days a week. Lately I put the stopwatch on and record the time it takes just so I can race my past self. It’s more exciting this way.

I’ve put off writing this post because I thought it’s “too girly”, but hey if boys can talk about cars and not being labeled “too guyish”, I don’t see why girls should shy away from topics like this. I hope you can share some of your experience too, I noticed quite a lot of my girl friends started putting serious makeup (using eyeshadows++) within this last year. We could have probably helped out each other and avoided the earlier frustrations.

About

I’m Min’an, the other couch potato from nimblecouchpotatoes.com . I spent most of my life in Singapore before picking up a degree in Sydney. where I met Marty, my other potato-half. I worked as a java developer/build engineer/firefighter at Atlassian for a few years before following Marty to Stockholm where she found the job of her dreams.

In my past life I worked a great deal with Bamboo and Maven, and had some involvement with JIRA. Firefighting takes its toll on you and I’m now on a journey to figure out what I had missed wearing the red suit.

Referencing local aar files with Android Studio’s new Gradle-based build system

The new Android Studio IDE uses a new, Gradle-based build system. If you want to reference other android libraries, the process is now a little more complicated . At face value, the docs suggest you need to

Keeping a copy of the library sources around is useful if you need to build it again. If you’re using git, git submodules or one of its alternatives would be useful for this.

OTOH, if the author has kindly uploaded it to a maven repository as an aar (an apklib generated by the android-maven-plugin will not work), you can consume it as a normal dependency, something like:

dependencies {
  compile 'com.github.chrisbanes.actionbarpulltorefresh:library:0.7.+'
}

If all you have is the aar itself though, you’ll be out of luck because local aar files are not supported. So something like:

dependencies {
  compile files('libs/some-awesome-library.aar')
}

will not work, only aar files obtained from say a maven repository are allowed.

It turns out that almost any non-file repository is allowed, even flatDirs, so

repositories {
  mavenCentral()
  flatDir {
    dirs 'libs'
  }
}
...
dependencies {
  compile 'com.actionbarsherlock:actionbarsherlock-local:4.4.0@aar'
}

will work, assuming that your local aars are found in the libs dir. Note the addition of @aar, which indicates the packaging type of the dependency. The group (com.actionbarsherlock in this case) doesn’t actually matter as the artifact name, version and packaging type are all that’s used to look up the aar file.

Whilst this does save you the trouble of having to maintain a dependency to another codebase, it means you’ll probably be checking in a magic, binary artifact. For libraries using the maven plugin to generate an apklib, you still need a build.gradle to produce the actual aar though I have found this to be rather straightforward.